hotspot/src/share/vm/opto/loopTransform.cpp
author kvn
Mon, 21 Mar 2011 11:28:14 -0700
changeset 8732 16fc1c68714b
parent 7397 5b173b4ca846
child 8870 119881dc9d0b
permissions -rw-r--r--
7008866: Missing loop predicate for loop with multiple entries Summary: Add predicates when loop head bytecode is parsed instead of when back branch bytecode is parsed. Reviewed-by: never
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
     1
/*
5547
f4b087cbb361 6941466: Oracle rebranding changes for Hotspot repositories
trims
parents: 5054
diff changeset
     2
 * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
     3
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
489c9b5090e2 Initial load
duke
parents:
diff changeset
     4
 *
489c9b5090e2 Initial load
duke
parents:
diff changeset
     5
 * This code is free software; you can redistribute it and/or modify it
489c9b5090e2 Initial load
duke
parents:
diff changeset
     6
 * under the terms of the GNU General Public License version 2 only, as
489c9b5090e2 Initial load
duke
parents:
diff changeset
     7
 * published by the Free Software Foundation.
489c9b5090e2 Initial load
duke
parents:
diff changeset
     8
 *
489c9b5090e2 Initial load
duke
parents:
diff changeset
     9
 * This code is distributed in the hope that it will be useful, but WITHOUT
489c9b5090e2 Initial load
duke
parents:
diff changeset
    10
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
489c9b5090e2 Initial load
duke
parents:
diff changeset
    11
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
489c9b5090e2 Initial load
duke
parents:
diff changeset
    12
 * version 2 for more details (a copy is included in the LICENSE file that
489c9b5090e2 Initial load
duke
parents:
diff changeset
    13
 * accompanied this code).
489c9b5090e2 Initial load
duke
parents:
diff changeset
    14
 *
489c9b5090e2 Initial load
duke
parents:
diff changeset
    15
 * You should have received a copy of the GNU General Public License version
489c9b5090e2 Initial load
duke
parents:
diff changeset
    16
 * 2 along with this work; if not, write to the Free Software Foundation,
489c9b5090e2 Initial load
duke
parents:
diff changeset
    17
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
489c9b5090e2 Initial load
duke
parents:
diff changeset
    18
 *
5547
f4b087cbb361 6941466: Oracle rebranding changes for Hotspot repositories
trims
parents: 5054
diff changeset
    19
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
f4b087cbb361 6941466: Oracle rebranding changes for Hotspot repositories
trims
parents: 5054
diff changeset
    20
 * or visit www.oracle.com if you need additional information or have any
f4b087cbb361 6941466: Oracle rebranding changes for Hotspot repositories
trims
parents: 5054
diff changeset
    21
 * questions.
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
    22
 *
489c9b5090e2 Initial load
duke
parents:
diff changeset
    23
 */
489c9b5090e2 Initial load
duke
parents:
diff changeset
    24
7397
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6770
diff changeset
    25
#include "precompiled.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6770
diff changeset
    26
#include "compiler/compileLog.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6770
diff changeset
    27
#include "memory/allocation.inline.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6770
diff changeset
    28
#include "opto/addnode.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6770
diff changeset
    29
#include "opto/callnode.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6770
diff changeset
    30
#include "opto/connode.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6770
diff changeset
    31
#include "opto/divnode.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6770
diff changeset
    32
#include "opto/loopnode.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6770
diff changeset
    33
#include "opto/mulnode.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6770
diff changeset
    34
#include "opto/rootnode.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6770
diff changeset
    35
#include "opto/runtime.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6770
diff changeset
    36
#include "opto/subnode.hpp"
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
    37
489c9b5090e2 Initial load
duke
parents:
diff changeset
    38
//------------------------------is_loop_exit-----------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
    39
// Given an IfNode, return the loop-exiting projection or NULL if both
489c9b5090e2 Initial load
duke
parents:
diff changeset
    40
// arms remain in the loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
    41
Node *IdealLoopTree::is_loop_exit(Node *iff) const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    42
  if( iff->outcnt() != 2 ) return NULL; // Ignore partially dead tests
489c9b5090e2 Initial load
duke
parents:
diff changeset
    43
  PhaseIdealLoop *phase = _phase;
489c9b5090e2 Initial load
duke
parents:
diff changeset
    44
  // Test is an IfNode, has 2 projections.  If BOTH are in the loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
    45
  // we need loop unswitching instead of peeling.
489c9b5090e2 Initial load
duke
parents:
diff changeset
    46
  if( !is_member(phase->get_loop( iff->raw_out(0) )) )
489c9b5090e2 Initial load
duke
parents:
diff changeset
    47
    return iff->raw_out(0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
    48
  if( !is_member(phase->get_loop( iff->raw_out(1) )) )
489c9b5090e2 Initial load
duke
parents:
diff changeset
    49
    return iff->raw_out(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
    50
  return NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
    51
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
    52
489c9b5090e2 Initial load
duke
parents:
diff changeset
    53
489c9b5090e2 Initial load
duke
parents:
diff changeset
    54
//=============================================================================
489c9b5090e2 Initial load
duke
parents:
diff changeset
    55
489c9b5090e2 Initial load
duke
parents:
diff changeset
    56
489c9b5090e2 Initial load
duke
parents:
diff changeset
    57
//------------------------------record_for_igvn----------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
    58
// Put loop body on igvn work list
489c9b5090e2 Initial load
duke
parents:
diff changeset
    59
void IdealLoopTree::record_for_igvn() {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    60
  for( uint i = 0; i < _body.size(); i++ ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    61
    Node *n = _body.at(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
    62
    _phase->_igvn._worklist.push(n);
489c9b5090e2 Initial load
duke
parents:
diff changeset
    63
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
    64
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
    65
489c9b5090e2 Initial load
duke
parents:
diff changeset
    66
//------------------------------compute_profile_trip_cnt----------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
    67
// Compute loop trip count from profile data as
489c9b5090e2 Initial load
duke
parents:
diff changeset
    68
//    (backedge_count + loop_exit_count) / loop_exit_count
489c9b5090e2 Initial load
duke
parents:
diff changeset
    69
void IdealLoopTree::compute_profile_trip_cnt( PhaseIdealLoop *phase ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    70
  if (!_head->is_CountedLoop()) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    71
    return;
489c9b5090e2 Initial load
duke
parents:
diff changeset
    72
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
    73
  CountedLoopNode* head = _head->as_CountedLoop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
    74
  if (head->profile_trip_cnt() != COUNT_UNKNOWN) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    75
    return; // Already computed
489c9b5090e2 Initial load
duke
parents:
diff changeset
    76
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
    77
  float trip_cnt = (float)max_jint; // default is big
489c9b5090e2 Initial load
duke
parents:
diff changeset
    78
489c9b5090e2 Initial load
duke
parents:
diff changeset
    79
  Node* back = head->in(LoopNode::LoopBackControl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
    80
  while (back != head) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    81
    if ((back->Opcode() == Op_IfTrue || back->Opcode() == Op_IfFalse) &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
    82
        back->in(0) &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
    83
        back->in(0)->is_If() &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
    84
        back->in(0)->as_If()->_fcnt != COUNT_UNKNOWN &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
    85
        back->in(0)->as_If()->_prob != PROB_UNKNOWN) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    86
      break;
489c9b5090e2 Initial load
duke
parents:
diff changeset
    87
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
    88
    back = phase->idom(back);
489c9b5090e2 Initial load
duke
parents:
diff changeset
    89
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
    90
  if (back != head) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    91
    assert((back->Opcode() == Op_IfTrue || back->Opcode() == Op_IfFalse) &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
    92
           back->in(0), "if-projection exists");
489c9b5090e2 Initial load
duke
parents:
diff changeset
    93
    IfNode* back_if = back->in(0)->as_If();
489c9b5090e2 Initial load
duke
parents:
diff changeset
    94
    float loop_back_cnt = back_if->_fcnt * back_if->_prob;
489c9b5090e2 Initial load
duke
parents:
diff changeset
    95
489c9b5090e2 Initial load
duke
parents:
diff changeset
    96
    // Now compute a loop exit count
489c9b5090e2 Initial load
duke
parents:
diff changeset
    97
    float loop_exit_cnt = 0.0f;
489c9b5090e2 Initial load
duke
parents:
diff changeset
    98
    for( uint i = 0; i < _body.size(); i++ ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    99
      Node *n = _body[i];
489c9b5090e2 Initial load
duke
parents:
diff changeset
   100
      if( n->is_If() ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   101
        IfNode *iff = n->as_If();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   102
        if( iff->_fcnt != COUNT_UNKNOWN && iff->_prob != PROB_UNKNOWN ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   103
          Node *exit = is_loop_exit(iff);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   104
          if( exit ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   105
            float exit_prob = iff->_prob;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   106
            if (exit->Opcode() == Op_IfFalse) exit_prob = 1.0 - exit_prob;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   107
            if (exit_prob > PROB_MIN) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   108
              float exit_cnt = iff->_fcnt * exit_prob;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   109
              loop_exit_cnt += exit_cnt;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   110
            }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   111
          }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   112
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   113
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   114
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   115
    if (loop_exit_cnt > 0.0f) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   116
      trip_cnt = (loop_back_cnt + loop_exit_cnt) / loop_exit_cnt;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   117
    } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   118
      // No exit count so use
489c9b5090e2 Initial load
duke
parents:
diff changeset
   119
      trip_cnt = loop_back_cnt;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   120
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   121
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   122
#ifndef PRODUCT
489c9b5090e2 Initial load
duke
parents:
diff changeset
   123
  if (TraceProfileTripCount) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   124
    tty->print_cr("compute_profile_trip_cnt  lp: %d cnt: %f\n", head->_idx, trip_cnt);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   125
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   126
#endif
489c9b5090e2 Initial load
duke
parents:
diff changeset
   127
  head->set_profile_trip_cnt(trip_cnt);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   128
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   129
489c9b5090e2 Initial load
duke
parents:
diff changeset
   130
//---------------------is_invariant_addition-----------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   131
// Return nonzero index of invariant operand for an Add or Sub
2131
98f9cef66a34 6810672: Comment typos
twisti
parents: 1500
diff changeset
   132
// of (nonconstant) invariant and variant values. Helper for reassociate_invariants.
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   133
int IdealLoopTree::is_invariant_addition(Node* n, PhaseIdealLoop *phase) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   134
  int op = n->Opcode();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   135
  if (op == Op_AddI || op == Op_SubI) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   136
    bool in1_invar = this->is_invariant(n->in(1));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   137
    bool in2_invar = this->is_invariant(n->in(2));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   138
    if (in1_invar && !in2_invar) return 1;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   139
    if (!in1_invar && in2_invar) return 2;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   140
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   141
  return 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   142
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   143
489c9b5090e2 Initial load
duke
parents:
diff changeset
   144
//---------------------reassociate_add_sub-----------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   145
// Reassociate invariant add and subtract expressions:
489c9b5090e2 Initial load
duke
parents:
diff changeset
   146
//
489c9b5090e2 Initial load
duke
parents:
diff changeset
   147
// inv1 + (x + inv2)  =>  ( inv1 + inv2) + x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   148
// (x + inv2) + inv1  =>  ( inv1 + inv2) + x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   149
// inv1 + (x - inv2)  =>  ( inv1 - inv2) + x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   150
// inv1 - (inv2 - x)  =>  ( inv1 - inv2) + x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   151
// (x + inv2) - inv1  =>  (-inv1 + inv2) + x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   152
// (x - inv2) + inv1  =>  ( inv1 - inv2) + x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   153
// (x - inv2) - inv1  =>  (-inv1 - inv2) + x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   154
// inv1 + (inv2 - x)  =>  ( inv1 + inv2) - x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   155
// inv1 - (x - inv2)  =>  ( inv1 + inv2) - x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   156
// (inv2 - x) + inv1  =>  ( inv1 + inv2) - x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   157
// (inv2 - x) - inv1  =>  (-inv1 + inv2) - x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   158
// inv1 - (x + inv2)  =>  ( inv1 - inv2) - x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   159
//
489c9b5090e2 Initial load
duke
parents:
diff changeset
   160
Node* IdealLoopTree::reassociate_add_sub(Node* n1, PhaseIdealLoop *phase) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   161
  if (!n1->is_Add() && !n1->is_Sub() || n1->outcnt() == 0) return NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   162
  if (is_invariant(n1)) return NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   163
  int inv1_idx = is_invariant_addition(n1, phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   164
  if (!inv1_idx) return NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   165
  // Don't mess with add of constant (igvn moves them to expression tree root.)
489c9b5090e2 Initial load
duke
parents:
diff changeset
   166
  if (n1->is_Add() && n1->in(2)->is_Con()) return NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   167
  Node* inv1 = n1->in(inv1_idx);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   168
  Node* n2 = n1->in(3 - inv1_idx);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   169
  int inv2_idx = is_invariant_addition(n2, phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   170
  if (!inv2_idx) return NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   171
  Node* x    = n2->in(3 - inv2_idx);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   172
  Node* inv2 = n2->in(inv2_idx);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   173
489c9b5090e2 Initial load
duke
parents:
diff changeset
   174
  bool neg_x    = n2->is_Sub() && inv2_idx == 1;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   175
  bool neg_inv2 = n2->is_Sub() && inv2_idx == 2;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   176
  bool neg_inv1 = n1->is_Sub() && inv1_idx == 2;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   177
  if (n1->is_Sub() && inv1_idx == 1) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   178
    neg_x    = !neg_x;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   179
    neg_inv2 = !neg_inv2;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   180
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   181
  Node* inv1_c = phase->get_ctrl(inv1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   182
  Node* inv2_c = phase->get_ctrl(inv2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   183
  Node* n_inv1;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   184
  if (neg_inv1) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   185
    Node *zero = phase->_igvn.intcon(0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   186
    phase->set_ctrl(zero, phase->C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   187
    n_inv1 = new (phase->C, 3) SubINode(zero, inv1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   188
    phase->register_new_node(n_inv1, inv1_c);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   189
  } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   190
    n_inv1 = inv1;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   191
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   192
  Node* inv;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   193
  if (neg_inv2) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   194
    inv = new (phase->C, 3) SubINode(n_inv1, inv2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   195
  } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   196
    inv = new (phase->C, 3) AddINode(n_inv1, inv2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   197
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   198
  phase->register_new_node(inv, phase->get_early_ctrl(inv));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   199
489c9b5090e2 Initial load
duke
parents:
diff changeset
   200
  Node* addx;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   201
  if (neg_x) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   202
    addx = new (phase->C, 3) SubINode(inv, x);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   203
  } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   204
    addx = new (phase->C, 3) AddINode(x, inv);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   205
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   206
  phase->register_new_node(addx, phase->get_ctrl(x));
5901
c046f8e9c52b 6677629: PhaseIterGVN::subsume_node() should call hash_delete() and add_users_to_worklist()
kvn
parents: 5547
diff changeset
   207
  phase->_igvn.replace_node(n1, addx);
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   208
  assert(phase->get_loop(phase->get_ctrl(n1)) == this, "");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   209
  _body.yank(n1);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   210
  return addx;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   211
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   212
489c9b5090e2 Initial load
duke
parents:
diff changeset
   213
//---------------------reassociate_invariants-----------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   214
// Reassociate invariant expressions:
489c9b5090e2 Initial load
duke
parents:
diff changeset
   215
void IdealLoopTree::reassociate_invariants(PhaseIdealLoop *phase) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   216
  for (int i = _body.size() - 1; i >= 0; i--) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   217
    Node *n = _body.at(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   218
    for (int j = 0; j < 5; j++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   219
      Node* nn = reassociate_add_sub(n, phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   220
      if (nn == NULL) break;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   221
      n = nn; // again
489c9b5090e2 Initial load
duke
parents:
diff changeset
   222
    };
489c9b5090e2 Initial load
duke
parents:
diff changeset
   223
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   224
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   225
489c9b5090e2 Initial load
duke
parents:
diff changeset
   226
//------------------------------policy_peeling---------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   227
// Return TRUE or FALSE if the loop should be peeled or not.  Peel if we can
489c9b5090e2 Initial load
duke
parents:
diff changeset
   228
// make some loop-invariant test (usually a null-check) happen before the loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   229
bool IdealLoopTree::policy_peeling( PhaseIdealLoop *phase ) const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   230
  Node *test = ((IdealLoopTree*)this)->tail();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   231
  int  body_size = ((IdealLoopTree*)this)->_body.size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   232
  int  uniq      = phase->C->unique();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   233
  // Peeling does loop cloning which can result in O(N^2) node construction
489c9b5090e2 Initial load
duke
parents:
diff changeset
   234
  if( body_size > 255 /* Prevent overflow for large body_size */
489c9b5090e2 Initial load
duke
parents:
diff changeset
   235
      || (body_size * body_size + uniq > MaxNodeLimit) ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   236
    return false;           // too large to safely clone
489c9b5090e2 Initial load
duke
parents:
diff changeset
   237
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   238
  while( test != _head ) {      // Scan till run off top of loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   239
    if( test->is_If() ) {       // Test?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   240
      Node *ctrl = phase->get_ctrl(test->in(1));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   241
      if (ctrl->is_top())
489c9b5090e2 Initial load
duke
parents:
diff changeset
   242
        return false;           // Found dead test on live IF?  No peeling!
489c9b5090e2 Initial load
duke
parents:
diff changeset
   243
      // Standard IF only has one input value to check for loop invariance
489c9b5090e2 Initial load
duke
parents:
diff changeset
   244
      assert( test->Opcode() == Op_If || test->Opcode() == Op_CountedLoopEnd, "Check this code when new subtype is added");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   245
      // Condition is not a member of this loop?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   246
      if( !is_member(phase->get_loop(ctrl)) &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   247
          is_loop_exit(test) )
489c9b5090e2 Initial load
duke
parents:
diff changeset
   248
        return true;            // Found reason to peel!
489c9b5090e2 Initial load
duke
parents:
diff changeset
   249
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   250
    // Walk up dominators to loop _head looking for test which is
489c9b5090e2 Initial load
duke
parents:
diff changeset
   251
    // executed on every path thru loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   252
    test = phase->idom(test);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   253
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   254
  return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   255
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   256
489c9b5090e2 Initial load
duke
parents:
diff changeset
   257
//------------------------------peeled_dom_test_elim---------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   258
// If we got the effect of peeling, either by actually peeling or by making
489c9b5090e2 Initial load
duke
parents:
diff changeset
   259
// a pre-loop which must execute at least once, we can remove all
489c9b5090e2 Initial load
duke
parents:
diff changeset
   260
// loop-invariant dominated tests in the main body.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   261
void PhaseIdealLoop::peeled_dom_test_elim( IdealLoopTree *loop, Node_List &old_new ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   262
  bool progress = true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   263
  while( progress ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   264
    progress = false;           // Reset for next iteration
489c9b5090e2 Initial load
duke
parents:
diff changeset
   265
    Node *prev = loop->_head->in(LoopNode::LoopBackControl);//loop->tail();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   266
    Node *test = prev->in(0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   267
    while( test != loop->_head ) { // Scan till run off top of loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   268
489c9b5090e2 Initial load
duke
parents:
diff changeset
   269
      int p_op = prev->Opcode();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   270
      if( (p_op == Op_IfFalse || p_op == Op_IfTrue) &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   271
          test->is_If() &&      // Test?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   272
          !test->in(1)->is_Con() && // And not already obvious?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   273
          // Condition is not a member of this loop?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   274
          !loop->is_member(get_loop(get_ctrl(test->in(1))))){
489c9b5090e2 Initial load
duke
parents:
diff changeset
   275
        // Walk loop body looking for instances of this test
489c9b5090e2 Initial load
duke
parents:
diff changeset
   276
        for( uint i = 0; i < loop->_body.size(); i++ ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   277
          Node *n = loop->_body.at(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   278
          if( n->is_If() && n->in(1) == test->in(1) /*&& n != loop->tail()->in(0)*/ ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   279
            // IfNode was dominated by version in peeled loop body
489c9b5090e2 Initial load
duke
parents:
diff changeset
   280
            progress = true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   281
            dominated_by( old_new[prev->_idx], n );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   282
          }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   283
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   284
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   285
      prev = test;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   286
      test = idom(test);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   287
    } // End of scan tests in loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   288
489c9b5090e2 Initial load
duke
parents:
diff changeset
   289
  } // End of while( progress )
489c9b5090e2 Initial load
duke
parents:
diff changeset
   290
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   291
489c9b5090e2 Initial load
duke
parents:
diff changeset
   292
//------------------------------do_peeling-------------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   293
// Peel the first iteration of the given loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   294
// Step 1: Clone the loop body.  The clone becomes the peeled iteration.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   295
//         The pre-loop illegally has 2 control users (old & new loops).
489c9b5090e2 Initial load
duke
parents:
diff changeset
   296
// Step 2: Make the old-loop fall-in edges point to the peeled iteration.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   297
//         Do this by making the old-loop fall-in edges act as if they came
489c9b5090e2 Initial load
duke
parents:
diff changeset
   298
//         around the loopback from the prior iteration (follow the old-loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   299
//         backedges) and then map to the new peeled iteration.  This leaves
489c9b5090e2 Initial load
duke
parents:
diff changeset
   300
//         the pre-loop with only 1 user (the new peeled iteration), but the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   301
//         peeled-loop backedge has 2 users.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   302
// Step 3: Cut the backedge on the clone (so its not a loop) and remove the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   303
//         extra backedge user.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   304
void PhaseIdealLoop::do_peeling( IdealLoopTree *loop, Node_List &old_new ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   305
489c9b5090e2 Initial load
duke
parents:
diff changeset
   306
  C->set_major_progress();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   307
  // Peeling a 'main' loop in a pre/main/post situation obfuscates the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   308
  // 'pre' loop from the main and the 'pre' can no longer have it's
489c9b5090e2 Initial load
duke
parents:
diff changeset
   309
  // iterations adjusted.  Therefore, we need to declare this loop as
489c9b5090e2 Initial load
duke
parents:
diff changeset
   310
  // no longer a 'main' loop; it will need new pre and post loops before
489c9b5090e2 Initial load
duke
parents:
diff changeset
   311
  // we can do further RCE.
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   312
#ifndef PRODUCT
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   313
  if (TraceLoopOpts) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   314
    tty->print("Peel         ");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   315
    loop->dump_head();
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   316
  }
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   317
#endif
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   318
  Node *h = loop->_head;
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   319
  if (h->is_CountedLoop()) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   320
    CountedLoopNode *cl = h->as_CountedLoop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   321
    assert(cl->trip_count() > 0, "peeling a fully unrolled loop");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   322
    cl->set_trip_count(cl->trip_count() - 1);
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   323
    if (cl->is_main_loop()) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   324
      cl->set_normal_loop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   325
#ifndef PRODUCT
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   326
      if (PrintOpto && VerifyLoopOptimizations) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   327
        tty->print("Peeling a 'main' loop; resetting to 'normal' ");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   328
        loop->dump_head();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   329
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   330
#endif
489c9b5090e2 Initial load
duke
parents:
diff changeset
   331
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   332
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   333
489c9b5090e2 Initial load
duke
parents:
diff changeset
   334
  // Step 1: Clone the loop body.  The clone becomes the peeled iteration.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   335
  //         The pre-loop illegally has 2 control users (old & new loops).
489c9b5090e2 Initial load
duke
parents:
diff changeset
   336
  clone_loop( loop, old_new, dom_depth(loop->_head) );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   337
489c9b5090e2 Initial load
duke
parents:
diff changeset
   338
489c9b5090e2 Initial load
duke
parents:
diff changeset
   339
  // Step 2: Make the old-loop fall-in edges point to the peeled iteration.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   340
  //         Do this by making the old-loop fall-in edges act as if they came
489c9b5090e2 Initial load
duke
parents:
diff changeset
   341
  //         around the loopback from the prior iteration (follow the old-loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   342
  //         backedges) and then map to the new peeled iteration.  This leaves
489c9b5090e2 Initial load
duke
parents:
diff changeset
   343
  //         the pre-loop with only 1 user (the new peeled iteration), but the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   344
  //         peeled-loop backedge has 2 users.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   345
  for (DUIterator_Fast jmax, j = loop->_head->fast_outs(jmax); j < jmax; j++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   346
    Node* old = loop->_head->fast_out(j);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   347
    if( old->in(0) == loop->_head && old->req() == 3 &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   348
        (old->is_Loop() || old->is_Phi()) ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   349
      Node *new_exit_value = old_new[old->in(LoopNode::LoopBackControl)->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
   350
      if( !new_exit_value )     // Backedge value is ALSO loop invariant?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   351
        // Then loop body backedge value remains the same.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   352
        new_exit_value = old->in(LoopNode::LoopBackControl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   353
      _igvn.hash_delete(old);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   354
      old->set_req(LoopNode::EntryControl, new_exit_value);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   355
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   356
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   357
489c9b5090e2 Initial load
duke
parents:
diff changeset
   358
489c9b5090e2 Initial load
duke
parents:
diff changeset
   359
  // Step 3: Cut the backedge on the clone (so its not a loop) and remove the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   360
  //         extra backedge user.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   361
  Node *nnn = old_new[loop->_head->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
   362
  _igvn.hash_delete(nnn);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   363
  nnn->set_req(LoopNode::LoopBackControl, C->top());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   364
  for (DUIterator_Fast j2max, j2 = nnn->fast_outs(j2max); j2 < j2max; j2++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   365
    Node* use = nnn->fast_out(j2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   366
    if( use->in(0) == nnn && use->req() == 3 && use->is_Phi() ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   367
      _igvn.hash_delete(use);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   368
      use->set_req(LoopNode::LoopBackControl, C->top());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   369
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   370
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   371
489c9b5090e2 Initial load
duke
parents:
diff changeset
   372
489c9b5090e2 Initial load
duke
parents:
diff changeset
   373
  // Step 4: Correct dom-depth info.  Set to loop-head depth.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   374
  int dd = dom_depth(loop->_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   375
  set_idom(loop->_head, loop->_head->in(1), dd);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   376
  for (uint j3 = 0; j3 < loop->_body.size(); j3++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   377
    Node *old = loop->_body.at(j3);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   378
    Node *nnn = old_new[old->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
   379
    if (!has_ctrl(nnn))
489c9b5090e2 Initial load
duke
parents:
diff changeset
   380
      set_idom(nnn, idom(nnn), dd-1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   381
    // While we're at it, remove any SafePoints from the peeled code
489c9b5090e2 Initial load
duke
parents:
diff changeset
   382
    if( old->Opcode() == Op_SafePoint ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   383
      Node *nnn = old_new[old->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
   384
      lazy_replace(nnn,nnn->in(TypeFunc::Control));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   385
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   386
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   387
489c9b5090e2 Initial load
duke
parents:
diff changeset
   388
  // Now force out all loop-invariant dominating tests.  The optimizer
489c9b5090e2 Initial load
duke
parents:
diff changeset
   389
  // finds some, but we _know_ they are all useless.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   390
  peeled_dom_test_elim(loop,old_new);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   391
489c9b5090e2 Initial load
duke
parents:
diff changeset
   392
  loop->record_for_igvn();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   393
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   394
489c9b5090e2 Initial load
duke
parents:
diff changeset
   395
//------------------------------policy_maximally_unroll------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   396
// Return exact loop trip count, or 0 if not maximally unrolling
489c9b5090e2 Initial load
duke
parents:
diff changeset
   397
bool IdealLoopTree::policy_maximally_unroll( PhaseIdealLoop *phase ) const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   398
  CountedLoopNode *cl = _head->as_CountedLoop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   399
  assert( cl->is_normal_loop(), "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   400
489c9b5090e2 Initial load
duke
parents:
diff changeset
   401
  Node *init_n = cl->init_trip();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   402
  Node *limit_n = cl->limit();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   403
489c9b5090e2 Initial load
duke
parents:
diff changeset
   404
  // Non-constant bounds
489c9b5090e2 Initial load
duke
parents:
diff changeset
   405
  if( init_n   == NULL || !init_n->is_Con()  ||
489c9b5090e2 Initial load
duke
parents:
diff changeset
   406
      limit_n  == NULL || !limit_n->is_Con() ||
489c9b5090e2 Initial load
duke
parents:
diff changeset
   407
      // protect against stride not being a constant
489c9b5090e2 Initial load
duke
parents:
diff changeset
   408
      !cl->stride_is_con() ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   409
    return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   410
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   411
  int init   = init_n->get_int();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   412
  int limit  = limit_n->get_int();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   413
  int span   = limit - init;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   414
  int stride = cl->stride_con();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   415
489c9b5090e2 Initial load
duke
parents:
diff changeset
   416
  if (init >= limit || stride > span) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   417
    // return a false (no maximally unroll) and the regular unroll/peel
489c9b5090e2 Initial load
duke
parents:
diff changeset
   418
    // route will make a small mess which CCP will fold away.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   419
    return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   420
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   421
  uint trip_count = span/stride;   // trip_count can be greater than 2 Gig.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   422
  assert( (int)trip_count*stride == span, "must divide evenly" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   423
489c9b5090e2 Initial load
duke
parents:
diff changeset
   424
  // Real policy: if we maximally unroll, does it get too big?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   425
  // Allow the unrolled mess to get larger than standard loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   426
  // size.  After all, it will no longer be a loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   427
  uint body_size    = _body.size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   428
  uint unroll_limit = (uint)LoopUnrollLimit * 4;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   429
  assert( (intx)unroll_limit == LoopUnrollLimit * 4, "LoopUnrollLimit must fit in 32bits");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   430
  cl->set_trip_count(trip_count);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   431
  if( trip_count <= unroll_limit && body_size <= unroll_limit ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   432
    uint new_body_size = body_size * trip_count;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   433
    if (new_body_size <= unroll_limit &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   434
        body_size == new_body_size / trip_count &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   435
        // Unrolling can result in a large amount of node construction
489c9b5090e2 Initial load
duke
parents:
diff changeset
   436
        new_body_size < MaxNodeLimit - phase->C->unique()) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   437
      return true;    // maximally unroll
489c9b5090e2 Initial load
duke
parents:
diff changeset
   438
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   439
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   440
489c9b5090e2 Initial load
duke
parents:
diff changeset
   441
  return false;               // Do not maximally unroll
489c9b5090e2 Initial load
duke
parents:
diff changeset
   442
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   443
489c9b5090e2 Initial load
duke
parents:
diff changeset
   444
489c9b5090e2 Initial load
duke
parents:
diff changeset
   445
//------------------------------policy_unroll----------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   446
// Return TRUE or FALSE if the loop should be unrolled or not.  Unroll if
489c9b5090e2 Initial load
duke
parents:
diff changeset
   447
// the loop is a CountedLoop and the body is small enough.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   448
bool IdealLoopTree::policy_unroll( PhaseIdealLoop *phase ) const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   449
489c9b5090e2 Initial load
duke
parents:
diff changeset
   450
  CountedLoopNode *cl = _head->as_CountedLoop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   451
  assert( cl->is_normal_loop() || cl->is_main_loop(), "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   452
489c9b5090e2 Initial load
duke
parents:
diff changeset
   453
  // protect against stride not being a constant
489c9b5090e2 Initial load
duke
parents:
diff changeset
   454
  if( !cl->stride_is_con() ) return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   455
489c9b5090e2 Initial load
duke
parents:
diff changeset
   456
  // protect against over-unrolling
489c9b5090e2 Initial load
duke
parents:
diff changeset
   457
  if( cl->trip_count() <= 1 ) return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   458
489c9b5090e2 Initial load
duke
parents:
diff changeset
   459
  int future_unroll_ct = cl->unrolled_count() * 2;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   460
489c9b5090e2 Initial load
duke
parents:
diff changeset
   461
  // Don't unroll if the next round of unrolling would push us
489c9b5090e2 Initial load
duke
parents:
diff changeset
   462
  // over the expected trip count of the loop.  One is subtracted
489c9b5090e2 Initial load
duke
parents:
diff changeset
   463
  // from the expected trip count because the pre-loop normally
489c9b5090e2 Initial load
duke
parents:
diff changeset
   464
  // executes 1 iteration.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   465
  if (UnrollLimitForProfileCheck > 0 &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   466
      cl->profile_trip_cnt() != COUNT_UNKNOWN &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   467
      future_unroll_ct        > UnrollLimitForProfileCheck &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   468
      (float)future_unroll_ct > cl->profile_trip_cnt() - 1.0) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   469
    return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   470
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   471
489c9b5090e2 Initial load
duke
parents:
diff changeset
   472
  // When unroll count is greater than LoopUnrollMin, don't unroll if:
489c9b5090e2 Initial load
duke
parents:
diff changeset
   473
  //   the residual iterations are more than 10% of the trip count
489c9b5090e2 Initial load
duke
parents:
diff changeset
   474
  //   and rounds of "unroll,optimize" are not making significant progress
489c9b5090e2 Initial load
duke
parents:
diff changeset
   475
  //   Progress defined as current size less than 20% larger than previous size.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   476
  if (UseSuperWord && cl->node_count_before_unroll() > 0 &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   477
      future_unroll_ct > LoopUnrollMin &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   478
      (future_unroll_ct - 1) * 10.0 > cl->profile_trip_cnt() &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   479
      1.2 * cl->node_count_before_unroll() < (double)_body.size()) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   480
    return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   481
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   482
489c9b5090e2 Initial load
duke
parents:
diff changeset
   483
  Node *init_n = cl->init_trip();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   484
  Node *limit_n = cl->limit();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   485
  // Non-constant bounds.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   486
  // Protect against over-unrolling when init or/and limit are not constant
489c9b5090e2 Initial load
duke
parents:
diff changeset
   487
  // (so that trip_count's init value is maxint) but iv range is known.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   488
  if( init_n   == NULL || !init_n->is_Con()  ||
489c9b5090e2 Initial load
duke
parents:
diff changeset
   489
      limit_n  == NULL || !limit_n->is_Con() ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   490
    Node* phi = cl->phi();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   491
    if( phi != NULL ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   492
      assert(phi->is_Phi() && phi->in(0) == _head, "Counted loop should have iv phi.");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   493
      const TypeInt* iv_type = phase->_igvn.type(phi)->is_int();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   494
      int next_stride = cl->stride_con() * 2; // stride after this unroll
489c9b5090e2 Initial load
duke
parents:
diff changeset
   495
      if( next_stride > 0 ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   496
        if( iv_type->_lo + next_stride <= iv_type->_lo || // overflow
489c9b5090e2 Initial load
duke
parents:
diff changeset
   497
            iv_type->_lo + next_stride >  iv_type->_hi ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   498
          return false;  // over-unrolling
489c9b5090e2 Initial load
duke
parents:
diff changeset
   499
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   500
      } else if( next_stride < 0 ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   501
        if( iv_type->_hi + next_stride >= iv_type->_hi || // overflow
489c9b5090e2 Initial load
duke
parents:
diff changeset
   502
            iv_type->_hi + next_stride <  iv_type->_lo ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   503
          return false;  // over-unrolling
489c9b5090e2 Initial load
duke
parents:
diff changeset
   504
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   505
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   506
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   507
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   508
489c9b5090e2 Initial load
duke
parents:
diff changeset
   509
  // Adjust body_size to determine if we unroll or not
489c9b5090e2 Initial load
duke
parents:
diff changeset
   510
  uint body_size = _body.size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   511
  // Key test to unroll CaffeineMark's Logic test
489c9b5090e2 Initial load
duke
parents:
diff changeset
   512
  int xors_in_loop = 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   513
  // Also count ModL, DivL and MulL which expand mightly
489c9b5090e2 Initial load
duke
parents:
diff changeset
   514
  for( uint k = 0; k < _body.size(); k++ ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   515
    switch( _body.at(k)->Opcode() ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   516
    case Op_XorI: xors_in_loop++; break; // CaffeineMark's Logic test
489c9b5090e2 Initial load
duke
parents:
diff changeset
   517
    case Op_ModL: body_size += 30; break;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   518
    case Op_DivL: body_size += 30; break;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   519
    case Op_MulL: body_size += 10; break;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   520
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   521
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   522
489c9b5090e2 Initial load
duke
parents:
diff changeset
   523
  // Check for being too big
489c9b5090e2 Initial load
duke
parents:
diff changeset
   524
  if( body_size > (uint)LoopUnrollLimit ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   525
    if( xors_in_loop >= 4 && body_size < (uint)LoopUnrollLimit*4) return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   526
    // Normal case: loop too big
489c9b5090e2 Initial load
duke
parents:
diff changeset
   527
    return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   528
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   529
489c9b5090e2 Initial load
duke
parents:
diff changeset
   530
  // Check for stride being a small enough constant
489c9b5090e2 Initial load
duke
parents:
diff changeset
   531
  if( abs(cl->stride_con()) > (1<<3) ) return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   532
489c9b5090e2 Initial load
duke
parents:
diff changeset
   533
  // Unroll once!  (Each trip will soon do double iterations)
489c9b5090e2 Initial load
duke
parents:
diff changeset
   534
  return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   535
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   536
489c9b5090e2 Initial load
duke
parents:
diff changeset
   537
//------------------------------policy_align-----------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   538
// Return TRUE or FALSE if the loop should be cache-line aligned.  Gather the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   539
// expression that does the alignment.  Note that only one array base can be
2131
98f9cef66a34 6810672: Comment typos
twisti
parents: 1500
diff changeset
   540
// aligned in a loop (unless the VM guarantees mutual alignment).  Note that
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   541
// if we vectorize short memory ops into longer memory ops, we may want to
489c9b5090e2 Initial load
duke
parents:
diff changeset
   542
// increase alignment.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   543
bool IdealLoopTree::policy_align( PhaseIdealLoop *phase ) const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   544
  return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   545
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   546
489c9b5090e2 Initial load
duke
parents:
diff changeset
   547
//------------------------------policy_range_check-----------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   548
// Return TRUE or FALSE if the loop should be range-check-eliminated.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   549
// Actually we do iteration-splitting, a more powerful form of RCE.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   550
bool IdealLoopTree::policy_range_check( PhaseIdealLoop *phase ) const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   551
  if( !RangeCheckElimination ) return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   552
489c9b5090e2 Initial load
duke
parents:
diff changeset
   553
  CountedLoopNode *cl = _head->as_CountedLoop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   554
  // If we unrolled with no intention of doing RCE and we later
489c9b5090e2 Initial load
duke
parents:
diff changeset
   555
  // changed our minds, we got no pre-loop.  Either we need to
489c9b5090e2 Initial load
duke
parents:
diff changeset
   556
  // make a new pre-loop, or we gotta disallow RCE.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   557
  if( cl->is_main_no_pre_loop() ) return false; // Disallowed for now.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   558
  Node *trip_counter = cl->phi();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   559
489c9b5090e2 Initial load
duke
parents:
diff changeset
   560
  // Check loop body for tests of trip-counter plus loop-invariant vs
489c9b5090e2 Initial load
duke
parents:
diff changeset
   561
  // loop-invariant.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   562
  for( uint i = 0; i < _body.size(); i++ ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   563
    Node *iff = _body[i];
489c9b5090e2 Initial load
duke
parents:
diff changeset
   564
    if( iff->Opcode() == Op_If ) { // Test?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   565
489c9b5090e2 Initial load
duke
parents:
diff changeset
   566
      // Comparing trip+off vs limit
489c9b5090e2 Initial load
duke
parents:
diff changeset
   567
      Node *bol = iff->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   568
      if( bol->req() != 2 ) continue; // dead constant test
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
   569
      if (!bol->is_Bool()) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
   570
        assert(UseLoopPredicate && bol->Opcode() == Op_Conv2B, "predicate check only");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
   571
        continue;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
   572
      }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   573
      Node *cmp = bol->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   574
489c9b5090e2 Initial load
duke
parents:
diff changeset
   575
      Node *rc_exp = cmp->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   576
      Node *limit = cmp->in(2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   577
489c9b5090e2 Initial load
duke
parents:
diff changeset
   578
      Node *limit_c = phase->get_ctrl(limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   579
      if( limit_c == phase->C->top() )
489c9b5090e2 Initial load
duke
parents:
diff changeset
   580
        return false;           // Found dead test on live IF?  No RCE!
489c9b5090e2 Initial load
duke
parents:
diff changeset
   581
      if( is_member(phase->get_loop(limit_c) ) ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   582
        // Compare might have operands swapped; commute them
489c9b5090e2 Initial load
duke
parents:
diff changeset
   583
        rc_exp = cmp->in(2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   584
        limit  = cmp->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   585
        limit_c = phase->get_ctrl(limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   586
        if( is_member(phase->get_loop(limit_c) ) )
489c9b5090e2 Initial load
duke
parents:
diff changeset
   587
          continue;             // Both inputs are loop varying; cannot RCE
489c9b5090e2 Initial load
duke
parents:
diff changeset
   588
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   589
489c9b5090e2 Initial load
duke
parents:
diff changeset
   590
      if (!phase->is_scaled_iv_plus_offset(rc_exp, trip_counter, NULL, NULL)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   591
        continue;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   592
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   593
      // Yeah!  Found a test like 'trip+off vs limit'
489c9b5090e2 Initial load
duke
parents:
diff changeset
   594
      // Test is an IfNode, has 2 projections.  If BOTH are in the loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   595
      // we need loop unswitching instead of iteration splitting.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   596
      if( is_loop_exit(iff) )
489c9b5090e2 Initial load
duke
parents:
diff changeset
   597
        return true;            // Found reason to split iterations
489c9b5090e2 Initial load
duke
parents:
diff changeset
   598
    } // End of is IF
489c9b5090e2 Initial load
duke
parents:
diff changeset
   599
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   600
489c9b5090e2 Initial load
duke
parents:
diff changeset
   601
  return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   602
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   603
489c9b5090e2 Initial load
duke
parents:
diff changeset
   604
//------------------------------policy_peel_only-------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   605
// Return TRUE or FALSE if the loop should NEVER be RCE'd or aligned.  Useful
489c9b5090e2 Initial load
duke
parents:
diff changeset
   606
// for unrolling loops with NO array accesses.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   607
bool IdealLoopTree::policy_peel_only( PhaseIdealLoop *phase ) const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   608
489c9b5090e2 Initial load
duke
parents:
diff changeset
   609
  for( uint i = 0; i < _body.size(); i++ )
489c9b5090e2 Initial load
duke
parents:
diff changeset
   610
    if( _body[i]->is_Mem() )
489c9b5090e2 Initial load
duke
parents:
diff changeset
   611
      return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   612
489c9b5090e2 Initial load
duke
parents:
diff changeset
   613
  // No memory accesses at all!
489c9b5090e2 Initial load
duke
parents:
diff changeset
   614
  return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   615
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   616
489c9b5090e2 Initial load
duke
parents:
diff changeset
   617
//------------------------------clone_up_backedge_goo--------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   618
// If Node n lives in the back_ctrl block and cannot float, we clone a private
489c9b5090e2 Initial load
duke
parents:
diff changeset
   619
// version of n in preheader_ctrl block and return that, otherwise return n.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   620
Node *PhaseIdealLoop::clone_up_backedge_goo( Node *back_ctrl, Node *preheader_ctrl, Node *n ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   621
  if( get_ctrl(n) != back_ctrl ) return n;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   622
489c9b5090e2 Initial load
duke
parents:
diff changeset
   623
  Node *x = NULL;               // If required, a clone of 'n'
489c9b5090e2 Initial load
duke
parents:
diff changeset
   624
  // Check for 'n' being pinned in the backedge.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   625
  if( n->in(0) && n->in(0) == back_ctrl ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   626
    x = n->clone();             // Clone a copy of 'n' to preheader
489c9b5090e2 Initial load
duke
parents:
diff changeset
   627
    x->set_req( 0, preheader_ctrl ); // Fix x's control input to preheader
489c9b5090e2 Initial load
duke
parents:
diff changeset
   628
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   629
489c9b5090e2 Initial load
duke
parents:
diff changeset
   630
  // Recursive fixup any other input edges into x.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   631
  // If there are no changes we can just return 'n', otherwise
489c9b5090e2 Initial load
duke
parents:
diff changeset
   632
  // we need to clone a private copy and change it.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   633
  for( uint i = 1; i < n->req(); i++ ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   634
    Node *g = clone_up_backedge_goo( back_ctrl, preheader_ctrl, n->in(i) );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   635
    if( g != n->in(i) ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   636
      if( !x )
489c9b5090e2 Initial load
duke
parents:
diff changeset
   637
        x = n->clone();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   638
      x->set_req(i, g);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   639
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   640
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   641
  if( x ) {                     // x can legally float to pre-header location
489c9b5090e2 Initial load
duke
parents:
diff changeset
   642
    register_new_node( x, preheader_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   643
    return x;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   644
  } else {                      // raise n to cover LCA of uses
489c9b5090e2 Initial load
duke
parents:
diff changeset
   645
    set_ctrl( n, find_non_split_ctrl(back_ctrl->in(0)) );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   646
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   647
  return n;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   648
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   649
489c9b5090e2 Initial load
duke
parents:
diff changeset
   650
//------------------------------insert_pre_post_loops--------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   651
// Insert pre and post loops.  If peel_only is set, the pre-loop can not have
489c9b5090e2 Initial load
duke
parents:
diff changeset
   652
// more iterations added.  It acts as a 'peel' only, no lower-bound RCE, no
489c9b5090e2 Initial load
duke
parents:
diff changeset
   653
// alignment.  Useful to unroll loops that do no array accesses.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   654
void PhaseIdealLoop::insert_pre_post_loops( IdealLoopTree *loop, Node_List &old_new, bool peel_only ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   655
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   656
#ifndef PRODUCT
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   657
  if (TraceLoopOpts) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   658
    if (peel_only)
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   659
      tty->print("PeelMainPost ");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   660
    else
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   661
      tty->print("PreMainPost  ");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   662
    loop->dump_head();
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   663
  }
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   664
#endif
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   665
  C->set_major_progress();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   666
489c9b5090e2 Initial load
duke
parents:
diff changeset
   667
  // Find common pieces of the loop being guarded with pre & post loops
489c9b5090e2 Initial load
duke
parents:
diff changeset
   668
  CountedLoopNode *main_head = loop->_head->as_CountedLoop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   669
  assert( main_head->is_normal_loop(), "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   670
  CountedLoopEndNode *main_end = main_head->loopexit();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   671
  assert( main_end->outcnt() == 2, "1 true, 1 false path only" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   672
  uint dd_main_head = dom_depth(main_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   673
  uint max = main_head->outcnt();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   674
489c9b5090e2 Initial load
duke
parents:
diff changeset
   675
  Node *pre_header= main_head->in(LoopNode::EntryControl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   676
  Node *init      = main_head->init_trip();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   677
  Node *incr      = main_end ->incr();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   678
  Node *limit     = main_end ->limit();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   679
  Node *stride    = main_end ->stride();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   680
  Node *cmp       = main_end ->cmp_node();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   681
  BoolTest::mask b_test = main_end->test_trip();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   682
489c9b5090e2 Initial load
duke
parents:
diff changeset
   683
  // Need only 1 user of 'bol' because I will be hacking the loop bounds.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   684
  Node *bol = main_end->in(CountedLoopEndNode::TestValue);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   685
  if( bol->outcnt() != 1 ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   686
    bol = bol->clone();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   687
    register_new_node(bol,main_end->in(CountedLoopEndNode::TestControl));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   688
    _igvn.hash_delete(main_end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   689
    main_end->set_req(CountedLoopEndNode::TestValue, bol);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   690
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   691
  // Need only 1 user of 'cmp' because I will be hacking the loop bounds.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   692
  if( cmp->outcnt() != 1 ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   693
    cmp = cmp->clone();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   694
    register_new_node(cmp,main_end->in(CountedLoopEndNode::TestControl));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   695
    _igvn.hash_delete(bol);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   696
    bol->set_req(1, cmp);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   697
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   698
489c9b5090e2 Initial load
duke
parents:
diff changeset
   699
  //------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   700
  // Step A: Create Post-Loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   701
  Node* main_exit = main_end->proj_out(false);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   702
  assert( main_exit->Opcode() == Op_IfFalse, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   703
  int dd_main_exit = dom_depth(main_exit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   704
489c9b5090e2 Initial load
duke
parents:
diff changeset
   705
  // Step A1: Clone the loop body.  The clone becomes the post-loop.  The main
489c9b5090e2 Initial load
duke
parents:
diff changeset
   706
  // loop pre-header illegally has 2 control users (old & new loops).
489c9b5090e2 Initial load
duke
parents:
diff changeset
   707
  clone_loop( loop, old_new, dd_main_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   708
  assert( old_new[main_end ->_idx]->Opcode() == Op_CountedLoopEnd, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   709
  CountedLoopNode *post_head = old_new[main_head->_idx]->as_CountedLoop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   710
  post_head->set_post_loop(main_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   711
1432
44f076e3d2a4 6667595: Set probability FAIR for pre-, post- loops and ALWAYS for main loop
kvn
parents: 1399
diff changeset
   712
  // Reduce the post-loop trip count.
44f076e3d2a4 6667595: Set probability FAIR for pre-, post- loops and ALWAYS for main loop
kvn
parents: 1399
diff changeset
   713
  CountedLoopEndNode* post_end = old_new[main_end ->_idx]->as_CountedLoopEnd();
44f076e3d2a4 6667595: Set probability FAIR for pre-, post- loops and ALWAYS for main loop
kvn
parents: 1399
diff changeset
   714
  post_end->_prob = PROB_FAIR;
44f076e3d2a4 6667595: Set probability FAIR for pre-, post- loops and ALWAYS for main loop
kvn
parents: 1399
diff changeset
   715
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   716
  // Build the main-loop normal exit.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   717
  IfFalseNode *new_main_exit = new (C, 1) IfFalseNode(main_end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   718
  _igvn.register_new_node_with_optimizer( new_main_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   719
  set_idom(new_main_exit, main_end, dd_main_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   720
  set_loop(new_main_exit, loop->_parent);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   721
489c9b5090e2 Initial load
duke
parents:
diff changeset
   722
  // Step A2: Build a zero-trip guard for the post-loop.  After leaving the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   723
  // main-loop, the post-loop may not execute at all.  We 'opaque' the incr
489c9b5090e2 Initial load
duke
parents:
diff changeset
   724
  // (the main-loop trip-counter exit value) because we will be changing
489c9b5090e2 Initial load
duke
parents:
diff changeset
   725
  // the exit value (via unrolling) so we cannot constant-fold away the zero
489c9b5090e2 Initial load
duke
parents:
diff changeset
   726
  // trip guard until all unrolling is done.
762
1b26adb5fea1 6715633: when matching a memory node the adr_type should not change
kvn
parents: 360
diff changeset
   727
  Node *zer_opaq = new (C, 2) Opaque1Node(C, incr);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   728
  Node *zer_cmp  = new (C, 3) CmpINode( zer_opaq, limit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   729
  Node *zer_bol  = new (C, 2) BoolNode( zer_cmp, b_test );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   730
  register_new_node( zer_opaq, new_main_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   731
  register_new_node( zer_cmp , new_main_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   732
  register_new_node( zer_bol , new_main_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   733
489c9b5090e2 Initial load
duke
parents:
diff changeset
   734
  // Build the IfNode
489c9b5090e2 Initial load
duke
parents:
diff changeset
   735
  IfNode *zer_iff = new (C, 2) IfNode( new_main_exit, zer_bol, PROB_FAIR, COUNT_UNKNOWN );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   736
  _igvn.register_new_node_with_optimizer( zer_iff );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   737
  set_idom(zer_iff, new_main_exit, dd_main_exit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   738
  set_loop(zer_iff, loop->_parent);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   739
489c9b5090e2 Initial load
duke
parents:
diff changeset
   740
  // Plug in the false-path, taken if we need to skip post-loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   741
  _igvn.hash_delete( main_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   742
  main_exit->set_req(0, zer_iff);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   743
  _igvn._worklist.push(main_exit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   744
  set_idom(main_exit, zer_iff, dd_main_exit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   745
  set_idom(main_exit->unique_out(), zer_iff, dd_main_exit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   746
  // Make the true-path, must enter the post loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   747
  Node *zer_taken = new (C, 1) IfTrueNode( zer_iff );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   748
  _igvn.register_new_node_with_optimizer( zer_taken );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   749
  set_idom(zer_taken, zer_iff, dd_main_exit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   750
  set_loop(zer_taken, loop->_parent);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   751
  // Plug in the true path
489c9b5090e2 Initial load
duke
parents:
diff changeset
   752
  _igvn.hash_delete( post_head );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   753
  post_head->set_req(LoopNode::EntryControl, zer_taken);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   754
  set_idom(post_head, zer_taken, dd_main_exit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   755
489c9b5090e2 Initial load
duke
parents:
diff changeset
   756
  // Step A3: Make the fall-in values to the post-loop come from the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   757
  // fall-out values of the main-loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   758
  for (DUIterator_Fast imax, i = main_head->fast_outs(imax); i < imax; i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   759
    Node* main_phi = main_head->fast_out(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   760
    if( main_phi->is_Phi() && main_phi->in(0) == main_head && main_phi->outcnt() >0 ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   761
      Node *post_phi = old_new[main_phi->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
   762
      Node *fallmain  = clone_up_backedge_goo(main_head->back_control(),
489c9b5090e2 Initial load
duke
parents:
diff changeset
   763
                                              post_head->init_control(),
489c9b5090e2 Initial load
duke
parents:
diff changeset
   764
                                              main_phi->in(LoopNode::LoopBackControl));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   765
      _igvn.hash_delete(post_phi);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   766
      post_phi->set_req( LoopNode::EntryControl, fallmain );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   767
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   768
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   769
489c9b5090e2 Initial load
duke
parents:
diff changeset
   770
  // Update local caches for next stanza
489c9b5090e2 Initial load
duke
parents:
diff changeset
   771
  main_exit = new_main_exit;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   772
489c9b5090e2 Initial load
duke
parents:
diff changeset
   773
489c9b5090e2 Initial load
duke
parents:
diff changeset
   774
  //------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   775
  // Step B: Create Pre-Loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   776
489c9b5090e2 Initial load
duke
parents:
diff changeset
   777
  // Step B1: Clone the loop body.  The clone becomes the pre-loop.  The main
489c9b5090e2 Initial load
duke
parents:
diff changeset
   778
  // loop pre-header illegally has 2 control users (old & new loops).
489c9b5090e2 Initial load
duke
parents:
diff changeset
   779
  clone_loop( loop, old_new, dd_main_head );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   780
  CountedLoopNode*    pre_head = old_new[main_head->_idx]->as_CountedLoop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   781
  CountedLoopEndNode* pre_end  = old_new[main_end ->_idx]->as_CountedLoopEnd();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   782
  pre_head->set_pre_loop(main_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   783
  Node *pre_incr = old_new[incr->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
   784
1432
44f076e3d2a4 6667595: Set probability FAIR for pre-, post- loops and ALWAYS for main loop
kvn
parents: 1399
diff changeset
   785
  // Reduce the pre-loop trip count.
44f076e3d2a4 6667595: Set probability FAIR for pre-, post- loops and ALWAYS for main loop
kvn
parents: 1399
diff changeset
   786
  pre_end->_prob = PROB_FAIR;
44f076e3d2a4 6667595: Set probability FAIR for pre-, post- loops and ALWAYS for main loop
kvn
parents: 1399
diff changeset
   787
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   788
  // Find the pre-loop normal exit.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   789
  Node* pre_exit = pre_end->proj_out(false);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   790
  assert( pre_exit->Opcode() == Op_IfFalse, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   791
  IfFalseNode *new_pre_exit = new (C, 1) IfFalseNode(pre_end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   792
  _igvn.register_new_node_with_optimizer( new_pre_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   793
  set_idom(new_pre_exit, pre_end, dd_main_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   794
  set_loop(new_pre_exit, loop->_parent);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   795
489c9b5090e2 Initial load
duke
parents:
diff changeset
   796
  // Step B2: Build a zero-trip guard for the main-loop.  After leaving the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   797
  // pre-loop, the main-loop may not execute at all.  Later in life this
489c9b5090e2 Initial load
duke
parents:
diff changeset
   798
  // zero-trip guard will become the minimum-trip guard when we unroll
489c9b5090e2 Initial load
duke
parents:
diff changeset
   799
  // the main-loop.
762
1b26adb5fea1 6715633: when matching a memory node the adr_type should not change
kvn
parents: 360
diff changeset
   800
  Node *min_opaq = new (C, 2) Opaque1Node(C, limit);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   801
  Node *min_cmp  = new (C, 3) CmpINode( pre_incr, min_opaq );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   802
  Node *min_bol  = new (C, 2) BoolNode( min_cmp, b_test );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   803
  register_new_node( min_opaq, new_pre_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   804
  register_new_node( min_cmp , new_pre_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   805
  register_new_node( min_bol , new_pre_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   806
1432
44f076e3d2a4 6667595: Set probability FAIR for pre-, post- loops and ALWAYS for main loop
kvn
parents: 1399
diff changeset
   807
  // Build the IfNode (assume the main-loop is executed always).
44f076e3d2a4 6667595: Set probability FAIR for pre-, post- loops and ALWAYS for main loop
kvn
parents: 1399
diff changeset
   808
  IfNode *min_iff = new (C, 2) IfNode( new_pre_exit, min_bol, PROB_ALWAYS, COUNT_UNKNOWN );
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   809
  _igvn.register_new_node_with_optimizer( min_iff );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   810
  set_idom(min_iff, new_pre_exit, dd_main_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   811
  set_loop(min_iff, loop->_parent);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   812
489c9b5090e2 Initial load
duke
parents:
diff changeset
   813
  // Plug in the false-path, taken if we need to skip main-loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   814
  _igvn.hash_delete( pre_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   815
  pre_exit->set_req(0, min_iff);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   816
  set_idom(pre_exit, min_iff, dd_main_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   817
  set_idom(pre_exit->unique_out(), min_iff, dd_main_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   818
  // Make the true-path, must enter the main loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   819
  Node *min_taken = new (C, 1) IfTrueNode( min_iff );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   820
  _igvn.register_new_node_with_optimizer( min_taken );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   821
  set_idom(min_taken, min_iff, dd_main_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   822
  set_loop(min_taken, loop->_parent);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   823
  // Plug in the true path
489c9b5090e2 Initial load
duke
parents:
diff changeset
   824
  _igvn.hash_delete( main_head );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   825
  main_head->set_req(LoopNode::EntryControl, min_taken);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   826
  set_idom(main_head, min_taken, dd_main_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   827
489c9b5090e2 Initial load
duke
parents:
diff changeset
   828
  // Step B3: Make the fall-in values to the main-loop come from the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   829
  // fall-out values of the pre-loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   830
  for (DUIterator_Fast i2max, i2 = main_head->fast_outs(i2max); i2 < i2max; i2++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   831
    Node* main_phi = main_head->fast_out(i2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   832
    if( main_phi->is_Phi() && main_phi->in(0) == main_head && main_phi->outcnt() > 0 ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   833
      Node *pre_phi = old_new[main_phi->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
   834
      Node *fallpre  = clone_up_backedge_goo(pre_head->back_control(),
489c9b5090e2 Initial load
duke
parents:
diff changeset
   835
                                             main_head->init_control(),
489c9b5090e2 Initial load
duke
parents:
diff changeset
   836
                                             pre_phi->in(LoopNode::LoopBackControl));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   837
      _igvn.hash_delete(main_phi);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   838
      main_phi->set_req( LoopNode::EntryControl, fallpre );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   839
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   840
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   841
489c9b5090e2 Initial load
duke
parents:
diff changeset
   842
  // Step B4: Shorten the pre-loop to run only 1 iteration (for now).
489c9b5090e2 Initial load
duke
parents:
diff changeset
   843
  // RCE and alignment may change this later.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   844
  Node *cmp_end = pre_end->cmp_node();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   845
  assert( cmp_end->in(2) == limit, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   846
  Node *pre_limit = new (C, 3) AddINode( init, stride );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   847
489c9b5090e2 Initial load
duke
parents:
diff changeset
   848
  // Save the original loop limit in this Opaque1 node for
489c9b5090e2 Initial load
duke
parents:
diff changeset
   849
  // use by range check elimination.
762
1b26adb5fea1 6715633: when matching a memory node the adr_type should not change
kvn
parents: 360
diff changeset
   850
  Node *pre_opaq  = new (C, 3) Opaque1Node(C, pre_limit, limit);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   851
489c9b5090e2 Initial load
duke
parents:
diff changeset
   852
  register_new_node( pre_limit, pre_head->in(0) );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   853
  register_new_node( pre_opaq , pre_head->in(0) );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   854
489c9b5090e2 Initial load
duke
parents:
diff changeset
   855
  // Since no other users of pre-loop compare, I can hack limit directly
489c9b5090e2 Initial load
duke
parents:
diff changeset
   856
  assert( cmp_end->outcnt() == 1, "no other users" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   857
  _igvn.hash_delete(cmp_end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   858
  cmp_end->set_req(2, peel_only ? pre_limit : pre_opaq);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   859
489c9b5090e2 Initial load
duke
parents:
diff changeset
   860
  // Special case for not-equal loop bounds:
489c9b5090e2 Initial load
duke
parents:
diff changeset
   861
  // Change pre loop test, main loop test, and the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   862
  // main loop guard test to use lt or gt depending on stride
489c9b5090e2 Initial load
duke
parents:
diff changeset
   863
  // direction:
489c9b5090e2 Initial load
duke
parents:
diff changeset
   864
  // positive stride use <
489c9b5090e2 Initial load
duke
parents:
diff changeset
   865
  // negative stride use >
489c9b5090e2 Initial load
duke
parents:
diff changeset
   866
489c9b5090e2 Initial load
duke
parents:
diff changeset
   867
  if (pre_end->in(CountedLoopEndNode::TestValue)->as_Bool()->_test._test == BoolTest::ne) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   868
489c9b5090e2 Initial load
duke
parents:
diff changeset
   869
    BoolTest::mask new_test = (main_end->stride_con() > 0) ? BoolTest::lt : BoolTest::gt;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   870
    // Modify pre loop end condition
489c9b5090e2 Initial load
duke
parents:
diff changeset
   871
    Node* pre_bol = pre_end->in(CountedLoopEndNode::TestValue)->as_Bool();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   872
    BoolNode* new_bol0 = new (C, 2) BoolNode(pre_bol->in(1), new_test);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   873
    register_new_node( new_bol0, pre_head->in(0) );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   874
    _igvn.hash_delete(pre_end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   875
    pre_end->set_req(CountedLoopEndNode::TestValue, new_bol0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   876
    // Modify main loop guard condition
489c9b5090e2 Initial load
duke
parents:
diff changeset
   877
    assert(min_iff->in(CountedLoopEndNode::TestValue) == min_bol, "guard okay");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   878
    BoolNode* new_bol1 = new (C, 2) BoolNode(min_bol->in(1), new_test);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   879
    register_new_node( new_bol1, new_pre_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   880
    _igvn.hash_delete(min_iff);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   881
    min_iff->set_req(CountedLoopEndNode::TestValue, new_bol1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   882
    // Modify main loop end condition
489c9b5090e2 Initial load
duke
parents:
diff changeset
   883
    BoolNode* main_bol = main_end->in(CountedLoopEndNode::TestValue)->as_Bool();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   884
    BoolNode* new_bol2 = new (C, 2) BoolNode(main_bol->in(1), new_test);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   885
    register_new_node( new_bol2, main_end->in(CountedLoopEndNode::TestControl) );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   886
    _igvn.hash_delete(main_end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   887
    main_end->set_req(CountedLoopEndNode::TestValue, new_bol2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   888
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   889
489c9b5090e2 Initial load
duke
parents:
diff changeset
   890
  // Flag main loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   891
  main_head->set_main_loop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   892
  if( peel_only ) main_head->set_main_no_pre_loop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   893
489c9b5090e2 Initial load
duke
parents:
diff changeset
   894
  // It's difficult to be precise about the trip-counts
489c9b5090e2 Initial load
duke
parents:
diff changeset
   895
  // for the pre/post loops.  They are usually very short,
489c9b5090e2 Initial load
duke
parents:
diff changeset
   896
  // so guess that 4 trips is a reasonable value.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   897
  post_head->set_profile_trip_cnt(4.0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   898
  pre_head->set_profile_trip_cnt(4.0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   899
489c9b5090e2 Initial load
duke
parents:
diff changeset
   900
  // Now force out all loop-invariant dominating tests.  The optimizer
489c9b5090e2 Initial load
duke
parents:
diff changeset
   901
  // finds some, but we _know_ they are all useless.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   902
  peeled_dom_test_elim(loop,old_new);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   903
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   904
489c9b5090e2 Initial load
duke
parents:
diff changeset
   905
//------------------------------is_invariant-----------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   906
// Return true if n is invariant
489c9b5090e2 Initial load
duke
parents:
diff changeset
   907
bool IdealLoopTree::is_invariant(Node* n) const {
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
   908
  Node *n_c = _phase->has_ctrl(n) ? _phase->get_ctrl(n) : n;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   909
  if (n_c->is_top()) return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   910
  return !is_member(_phase->get_loop(n_c));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   911
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   912
489c9b5090e2 Initial load
duke
parents:
diff changeset
   913
489c9b5090e2 Initial load
duke
parents:
diff changeset
   914
//------------------------------do_unroll--------------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   915
// Unroll the loop body one step - make each trip do 2 iterations.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   916
void PhaseIdealLoop::do_unroll( IdealLoopTree *loop, Node_List &old_new, bool adjust_min_trip ) {
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   917
  assert(LoopUnrollLimit, "");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   918
  CountedLoopNode *loop_head = loop->_head->as_CountedLoop();
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   919
  CountedLoopEndNode *loop_end = loop_head->loopexit();
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   920
  assert(loop_end, "");
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   921
#ifndef PRODUCT
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   922
  if (PrintOpto && VerifyLoopOptimizations) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   923
    tty->print("Unrolling ");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   924
    loop->dump_head();
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   925
  } else if (TraceLoopOpts) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   926
    tty->print("Unroll     %d ", loop_head->unrolled_count()*2);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   927
    loop->dump_head();
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   928
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   929
#endif
489c9b5090e2 Initial load
duke
parents:
diff changeset
   930
489c9b5090e2 Initial load
duke
parents:
diff changeset
   931
  // Remember loop node count before unrolling to detect
489c9b5090e2 Initial load
duke
parents:
diff changeset
   932
  // if rounds of unroll,optimize are making progress
489c9b5090e2 Initial load
duke
parents:
diff changeset
   933
  loop_head->set_node_count_before_unroll(loop->_body.size());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   934
489c9b5090e2 Initial load
duke
parents:
diff changeset
   935
  Node *ctrl  = loop_head->in(LoopNode::EntryControl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   936
  Node *limit = loop_head->limit();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   937
  Node *init  = loop_head->init_trip();
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   938
  Node *stride = loop_head->stride();
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   939
489c9b5090e2 Initial load
duke
parents:
diff changeset
   940
  Node *opaq = NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   941
  if( adjust_min_trip ) {       // If not maximally unrolling, need adjustment
489c9b5090e2 Initial load
duke
parents:
diff changeset
   942
    assert( loop_head->is_main_loop(), "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   943
    assert( ctrl->Opcode() == Op_IfTrue || ctrl->Opcode() == Op_IfFalse, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   944
    Node *iff = ctrl->in(0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   945
    assert( iff->Opcode() == Op_If, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   946
    Node *bol = iff->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   947
    assert( bol->Opcode() == Op_Bool, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   948
    Node *cmp = bol->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   949
    assert( cmp->Opcode() == Op_CmpI, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   950
    opaq = cmp->in(2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   951
    // Occasionally it's possible for a pre-loop Opaque1 node to be
489c9b5090e2 Initial load
duke
parents:
diff changeset
   952
    // optimized away and then another round of loop opts attempted.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   953
    // We can not optimize this particular loop in that case.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   954
    if( opaq->Opcode() != Op_Opaque1 )
489c9b5090e2 Initial load
duke
parents:
diff changeset
   955
      return;                   // Cannot find pre-loop!  Bail out!
489c9b5090e2 Initial load
duke
parents:
diff changeset
   956
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   957
489c9b5090e2 Initial load
duke
parents:
diff changeset
   958
  C->set_major_progress();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   959
489c9b5090e2 Initial load
duke
parents:
diff changeset
   960
  // Adjust max trip count. The trip count is intentionally rounded
489c9b5090e2 Initial load
duke
parents:
diff changeset
   961
  // down here (e.g. 15-> 7-> 3-> 1) because if we unwittingly over-unroll,
489c9b5090e2 Initial load
duke
parents:
diff changeset
   962
  // the main, unrolled, part of the loop will never execute as it is protected
489c9b5090e2 Initial load
duke
parents:
diff changeset
   963
  // by the min-trip test.  See bug 4834191 for a case where we over-unrolled
489c9b5090e2 Initial load
duke
parents:
diff changeset
   964
  // and later determined that part of the unrolled loop was dead.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   965
  loop_head->set_trip_count(loop_head->trip_count() / 2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   966
489c9b5090e2 Initial load
duke
parents:
diff changeset
   967
  // Double the count of original iterations in the unrolled loop body.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   968
  loop_head->double_unrolled_count();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   969
489c9b5090e2 Initial load
duke
parents:
diff changeset
   970
  // -----------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   971
  // Step 2: Cut back the trip counter for an unroll amount of 2.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   972
  // Loop will normally trip (limit - init)/stride_con.  Since it's a
489c9b5090e2 Initial load
duke
parents:
diff changeset
   973
  // CountedLoop this is exact (stride divides limit-init exactly).
489c9b5090e2 Initial load
duke
parents:
diff changeset
   974
  // We are going to double the loop body, so we want to knock off any
489c9b5090e2 Initial load
duke
parents:
diff changeset
   975
  // odd iteration: (trip_cnt & ~1).  Then back compute a new limit.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   976
  Node *span = new (C, 3) SubINode( limit, init );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   977
  register_new_node( span, ctrl );
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   978
  Node *trip = new (C, 3) DivINode( 0, span, stride );
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   979
  register_new_node( trip, ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   980
  Node *mtwo = _igvn.intcon(-2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   981
  set_ctrl(mtwo, C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   982
  Node *rond = new (C, 3) AndINode( trip, mtwo );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   983
  register_new_node( rond, ctrl );
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   984
  Node *spn2 = new (C, 3) MulINode( rond, stride );
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   985
  register_new_node( spn2, ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   986
  Node *lim2 = new (C, 3) AddINode( spn2, init );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   987
  register_new_node( lim2, ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   988
489c9b5090e2 Initial load
duke
parents:
diff changeset
   989
  // Hammer in the new limit
489c9b5090e2 Initial load
duke
parents:
diff changeset
   990
  Node *ctrl2 = loop_end->in(0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   991
  Node *cmp2 = new (C, 3) CmpINode( loop_head->incr(), lim2 );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   992
  register_new_node( cmp2, ctrl2 );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   993
  Node *bol2 = new (C, 2) BoolNode( cmp2, loop_end->test_trip() );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   994
  register_new_node( bol2, ctrl2 );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   995
  _igvn.hash_delete(loop_end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   996
  loop_end->set_req(CountedLoopEndNode::TestValue, bol2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   997
489c9b5090e2 Initial load
duke
parents:
diff changeset
   998
  // Step 3: Find the min-trip test guaranteed before a 'main' loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   999
  // Make it a 1-trip test (means at least 2 trips).
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1000
  if( adjust_min_trip ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1001
    // Guard test uses an 'opaque' node which is not shared.  Hence I
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1002
    // can edit it's inputs directly.  Hammer in the new limit for the
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1003
    // minimum-trip guard.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1004
    assert( opaq->outcnt() == 1, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1005
    _igvn.hash_delete(opaq);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1006
    opaq->set_req(1, lim2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1007
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1008
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1009
  // ---------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1010
  // Step 4: Clone the loop body.  Move it inside the loop.  This loop body
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1011
  // represents the odd iterations; since the loop trips an even number of
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1012
  // times its backedge is never taken.  Kill the backedge.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1013
  uint dd = dom_depth(loop_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1014
  clone_loop( loop, old_new, dd );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1015
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1016
  // Make backedges of the clone equal to backedges of the original.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1017
  // Make the fall-in from the original come from the fall-out of the clone.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1018
  for (DUIterator_Fast jmax, j = loop_head->fast_outs(jmax); j < jmax; j++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1019
    Node* phi = loop_head->fast_out(j);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1020
    if( phi->is_Phi() && phi->in(0) == loop_head && phi->outcnt() > 0 ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1021
      Node *newphi = old_new[phi->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1022
      _igvn.hash_delete( phi );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1023
      _igvn.hash_delete( newphi );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1024
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1025
      phi   ->set_req(LoopNode::   EntryControl, newphi->in(LoopNode::LoopBackControl));
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1026
      newphi->set_req(LoopNode::LoopBackControl, phi   ->in(LoopNode::LoopBackControl));
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1027
      phi   ->set_req(LoopNode::LoopBackControl, C->top());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1028
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1029
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1030
  Node *clone_head = old_new[loop_head->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1031
  _igvn.hash_delete( clone_head );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1032
  loop_head ->set_req(LoopNode::   EntryControl, clone_head->in(LoopNode::LoopBackControl));
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1033
  clone_head->set_req(LoopNode::LoopBackControl, loop_head ->in(LoopNode::LoopBackControl));
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1034
  loop_head ->set_req(LoopNode::LoopBackControl, C->top());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1035
  loop->_head = clone_head;     // New loop header
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1036
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1037
  set_idom(loop_head,  loop_head ->in(LoopNode::EntryControl), dd);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1038
  set_idom(clone_head, clone_head->in(LoopNode::EntryControl), dd);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1039
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1040
  // Kill the clone's backedge
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1041
  Node *newcle = old_new[loop_end->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1042
  _igvn.hash_delete( newcle );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1043
  Node *one = _igvn.intcon(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1044
  set_ctrl(one, C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1045
  newcle->set_req(1, one);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1046
  // Force clone into same loop body
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1047
  uint max = loop->_body.size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1048
  for( uint k = 0; k < max; k++ ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1049
    Node *old = loop->_body.at(k);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1050
    Node *nnn = old_new[old->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1051
    loop->_body.push(nnn);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1052
    if (!has_ctrl(old))
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1053
      set_loop(nnn, loop);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1054
  }
1399
9648dfd4ce09 6384206: Phis which are later unneeded are impairing our ability to inline based on static types
never
parents: 781
diff changeset
  1055
9648dfd4ce09 6384206: Phis which are later unneeded are impairing our ability to inline based on static types
never
parents: 781
diff changeset
  1056
  loop->record_for_igvn();
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1057
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1058
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1059
//------------------------------do_maximally_unroll----------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1060
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1061
void PhaseIdealLoop::do_maximally_unroll( IdealLoopTree *loop, Node_List &old_new ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1062
  CountedLoopNode *cl = loop->_head->as_CountedLoop();
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1063
  assert(cl->trip_count() > 0, "");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1064
#ifndef PRODUCT
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1065
  if (TraceLoopOpts) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1066
    tty->print("MaxUnroll  %d ", cl->trip_count());
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1067
    loop->dump_head();
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1068
  }
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1069
#endif
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1070
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1071
  // If loop is tripping an odd number of times, peel odd iteration
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1072
  if ((cl->trip_count() & 1) == 1) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1073
    do_peeling(loop, old_new);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1074
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1075
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1076
  // Now its tripping an even number of times remaining.  Double loop body.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1077
  // Do not adjust pre-guards; they are not needed and do not exist.
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1078
  if (cl->trip_count() > 0) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1079
    do_unroll(loop, old_new, false);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1080
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1081
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1082
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1083
//------------------------------dominates_backedge---------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1084
// Returns true if ctrl is executed on every complete iteration
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1085
bool IdealLoopTree::dominates_backedge(Node* ctrl) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1086
  assert(ctrl->is_CFG(), "must be control");
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1087
  Node* backedge = _head->as_Loop()->in(LoopNode::LoopBackControl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1088
  return _phase->dom_lca_internal(ctrl, backedge) == ctrl;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1089
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1090
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1091
//------------------------------add_constraint---------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1092
// Constrain the main loop iterations so the condition:
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1093
//    scale_con * I + offset  <  limit
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1094
// always holds true.  That is, either increase the number of iterations in
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1095
// the pre-loop or the post-loop until the condition holds true in the main
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1096
// loop.  Stride, scale, offset and limit are all loop invariant.  Further,
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1097
// stride and scale are constants (offset and limit often are).
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1098
void PhaseIdealLoop::add_constraint( int stride_con, int scale_con, Node *offset, Node *limit, Node *pre_ctrl, Node **pre_limit, Node **main_limit ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1099
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1100
  // Compute "I :: (limit-offset)/scale_con"
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1101
  Node *con = new (C, 3) SubINode( limit, offset );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1102
  register_new_node( con, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1103
  Node *scale = _igvn.intcon(scale_con);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1104
  set_ctrl(scale, C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1105
  Node *X = new (C, 3) DivINode( 0, con, scale );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1106
  register_new_node( X, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1107
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1108
  // For positive stride, the pre-loop limit always uses a MAX function
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1109
  // and the main loop a MIN function.  For negative stride these are
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1110
  // reversed.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1111
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1112
  // Also for positive stride*scale the affine function is increasing, so the
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1113
  // pre-loop must check for underflow and the post-loop for overflow.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1114
  // Negative stride*scale reverses this; pre-loop checks for overflow and
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1115
  // post-loop for underflow.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1116
  if( stride_con*scale_con > 0 ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1117
    // Compute I < (limit-offset)/scale_con
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1118
    // Adjust main-loop last iteration to be MIN/MAX(main_loop,X)
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1119
    *main_limit = (stride_con > 0)
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1120
      ? (Node*)(new (C, 3) MinINode( *main_limit, X ))
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1121
      : (Node*)(new (C, 3) MaxINode( *main_limit, X ));
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1122
    register_new_node( *main_limit, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1123
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1124
  } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1125
    // Compute (limit-offset)/scale_con + SGN(-scale_con) <= I
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1126
    // Add the negation of the main-loop constraint to the pre-loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1127
    // See footnote [++] below for a derivation of the limit expression.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1128
    Node *incr = _igvn.intcon(scale_con > 0 ? -1 : 1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1129
    set_ctrl(incr, C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1130
    Node *adj = new (C, 3) AddINode( X, incr );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1131
    register_new_node( adj, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1132
    *pre_limit = (scale_con > 0)
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1133
      ? (Node*)new (C, 3) MinINode( *pre_limit, adj )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1134
      : (Node*)new (C, 3) MaxINode( *pre_limit, adj );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1135
    register_new_node( *pre_limit, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1136
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1137
//   [++] Here's the algebra that justifies the pre-loop limit expression:
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1138
//
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1139
//   NOT( scale_con * I + offset  <  limit )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1140
//      ==
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1141
//   scale_con * I + offset  >=  limit
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1142
//      ==
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1143
//   SGN(scale_con) * I  >=  (limit-offset)/|scale_con|
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1144
//      ==
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1145
//   (limit-offset)/|scale_con|   <=  I * SGN(scale_con)
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1146
//      ==
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1147
//   (limit-offset)/|scale_con|-1  <  I * SGN(scale_con)
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1148
//      ==
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1149
//   ( if (scale_con > 0) /*common case*/
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1150
//       (limit-offset)/scale_con - 1  <  I
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1151
//     else
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1152
//       (limit-offset)/scale_con + 1  >  I
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1153
//    )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1154
//   ( if (scale_con > 0) /*common case*/
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1155
//       (limit-offset)/scale_con + SGN(-scale_con)  <  I
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1156
//     else
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1157
//       (limit-offset)/scale_con + SGN(-scale_con)  >  I
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1158
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1159
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1160
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1161
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1162
//------------------------------is_scaled_iv---------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1163
// Return true if exp is a constant times an induction var
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1164
bool PhaseIdealLoop::is_scaled_iv(Node* exp, Node* iv, int* p_scale) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1165
  if (exp == iv) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1166
    if (p_scale != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1167
      *p_scale = 1;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1168
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1169
    return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1170
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1171
  int opc = exp->Opcode();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1172
  if (opc == Op_MulI) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1173
    if (exp->in(1) == iv && exp->in(2)->is_Con()) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1174
      if (p_scale != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1175
        *p_scale = exp->in(2)->get_int();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1176
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1177
      return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1178
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1179
    if (exp->in(2) == iv && exp->in(1)->is_Con()) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1180
      if (p_scale != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1181
        *p_scale = exp->in(1)->get_int();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1182
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1183
      return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1184
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1185
  } else if (opc == Op_LShiftI) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1186
    if (exp->in(1) == iv && exp->in(2)->is_Con()) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1187
      if (p_scale != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1188
        *p_scale = 1 << exp->in(2)->get_int();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1189
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1190
      return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1191
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1192
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1193
  return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1194
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1195
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1196
//-----------------------------is_scaled_iv_plus_offset------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1197
// Return true if exp is a simple induction variable expression: k1*iv + (invar + k2)
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1198
bool PhaseIdealLoop::is_scaled_iv_plus_offset(Node* exp, Node* iv, int* p_scale, Node** p_offset, int depth) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1199
  if (is_scaled_iv(exp, iv, p_scale)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1200
    if (p_offset != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1201
      Node *zero = _igvn.intcon(0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1202
      set_ctrl(zero, C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1203
      *p_offset = zero;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1204
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1205
    return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1206
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1207
  int opc = exp->Opcode();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1208
  if (opc == Op_AddI) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1209
    if (is_scaled_iv(exp->in(1), iv, p_scale)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1210
      if (p_offset != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1211
        *p_offset = exp->in(2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1212
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1213
      return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1214
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1215
    if (exp->in(2)->is_Con()) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1216
      Node* offset2 = NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1217
      if (depth < 2 &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1218
          is_scaled_iv_plus_offset(exp->in(1), iv, p_scale,
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1219
                                   p_offset != NULL ? &offset2 : NULL, depth+1)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1220
        if (p_offset != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1221
          Node *ctrl_off2 = get_ctrl(offset2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1222
          Node* offset = new (C, 3) AddINode(offset2, exp->in(2));
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1223
          register_new_node(offset, ctrl_off2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1224
          *p_offset = offset;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1225
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1226
        return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1227
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1228
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1229
  } else if (opc == Op_SubI) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1230
    if (is_scaled_iv(exp->in(1), iv, p_scale)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1231
      if (p_offset != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1232
        Node *zero = _igvn.intcon(0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1233
        set_ctrl(zero, C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1234
        Node *ctrl_off = get_ctrl(exp->in(2));
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1235
        Node* offset = new (C, 3) SubINode(zero, exp->in(2));
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1236
        register_new_node(offset, ctrl_off);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1237
        *p_offset = offset;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1238
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1239
      return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1240
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1241
    if (is_scaled_iv(exp->in(2), iv, p_scale)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1242
      if (p_offset != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1243
        *p_scale *= -1;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1244
        *p_offset = exp->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1245
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1246
      return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1247
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1248
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1249
  return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1250
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1251
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1252
//------------------------------do_range_check---------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1253
// Eliminate range-checks and other trip-counter vs loop-invariant tests.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1254
void PhaseIdealLoop::do_range_check( IdealLoopTree *loop, Node_List &old_new ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1255
#ifndef PRODUCT
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1256
  if (PrintOpto && VerifyLoopOptimizations) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1257
    tty->print("Range Check Elimination ");
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1258
    loop->dump_head();
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1259
  } else if (TraceLoopOpts) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1260
    tty->print("RangeCheck   ");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1261
    loop->dump_head();
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1262
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1263
#endif
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1264
  assert(RangeCheckElimination, "");
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1265
  CountedLoopNode *cl = loop->_head->as_CountedLoop();
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1266
  assert(cl->is_main_loop(), "");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1267
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1268
  // protect against stride not being a constant
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1269
  if (!cl->stride_is_con())
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1270
    return;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1271
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1272
  // Find the trip counter; we are iteration splitting based on it
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1273
  Node *trip_counter = cl->phi();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1274
  // Find the main loop limit; we will trim it's iterations
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1275
  // to not ever trip end tests
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1276
  Node *main_limit = cl->limit();
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1277
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1278
  // Need to find the main-loop zero-trip guard
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1279
  Node *ctrl  = cl->in(LoopNode::EntryControl);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1280
  assert(ctrl->Opcode() == Op_IfTrue || ctrl->Opcode() == Op_IfFalse, "");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1281
  Node *iffm = ctrl->in(0);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1282
  assert(iffm->Opcode() == Op_If, "");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1283
  Node *bolzm = iffm->in(1);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1284
  assert(bolzm->Opcode() == Op_Bool, "");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1285
  Node *cmpzm = bolzm->in(1);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1286
  assert(cmpzm->is_Cmp(), "");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1287
  Node *opqzm = cmpzm->in(2);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1288
  // Can not optimize a loop if pre-loop Opaque1 node is optimized
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1289
  // away and then another round of loop opts attempted.
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1290
  if (opqzm->Opcode() != Op_Opaque1)
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1291
    return;
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1292
  assert(opqzm->in(1) == main_limit, "do not understand situation");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1293
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1294
  // Find the pre-loop limit; we will expand it's iterations to
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1295
  // not ever trip low tests.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1296
  Node *p_f = iffm->in(0);
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1297
  assert(p_f->Opcode() == Op_IfFalse, "");
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1298
  CountedLoopEndNode *pre_end = p_f->in(0)->as_CountedLoopEnd();
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1299
  assert(pre_end->loopnode()->is_pre_loop(), "");
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1300
  Node *pre_opaq1 = pre_end->limit();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1301
  // Occasionally it's possible for a pre-loop Opaque1 node to be
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1302
  // optimized away and then another round of loop opts attempted.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1303
  // We can not optimize this particular loop in that case.
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1304
  if (pre_opaq1->Opcode() != Op_Opaque1)
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1305
    return;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1306
  Opaque1Node *pre_opaq = (Opaque1Node*)pre_opaq1;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1307
  Node *pre_limit = pre_opaq->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1308
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1309
  // Where do we put new limit calculations
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1310
  Node *pre_ctrl = pre_end->loopnode()->in(LoopNode::EntryControl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1311
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1312
  // Ensure the original loop limit is available from the
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1313
  // pre-loop Opaque1 node.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1314
  Node *orig_limit = pre_opaq->original_loop_limit();
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1315
  if (orig_limit == NULL || _igvn.type(orig_limit) == Type::TOP)
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1316
    return;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1317
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1318
  // Must know if its a count-up or count-down loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1319
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1320
  int stride_con = cl->stride_con();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1321
  Node *zero = _igvn.intcon(0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1322
  Node *one  = _igvn.intcon(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1323
  set_ctrl(zero, C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1324
  set_ctrl(one,  C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1325
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1326
  // Range checks that do not dominate the loop backedge (ie.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1327
  // conditionally executed) can lengthen the pre loop limit beyond
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1328
  // the original loop limit. To prevent this, the pre limit is
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1329
  // (for stride > 0) MINed with the original loop limit (MAXed
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1330
  // stride < 0) when some range_check (rc) is conditionally
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1331
  // executed.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1332
  bool conditional_rc = false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1333
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1334
  // Check loop body for tests of trip-counter plus loop-invariant vs
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1335
  // loop-invariant.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1336
  for( uint i = 0; i < loop->_body.size(); i++ ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1337
    Node *iff = loop->_body[i];
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1338
    if( iff->Opcode() == Op_If ) { // Test?
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1339
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1340
      // Test is an IfNode, has 2 projections.  If BOTH are in the loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1341
      // we need loop unswitching instead of iteration splitting.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1342
      Node *exit = loop->is_loop_exit(iff);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1343
      if( !exit ) continue;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1344
      int flip = (exit->Opcode() == Op_IfTrue) ? 1 : 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1345
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1346
      // Get boolean condition to test
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1347
      Node *i1 = iff->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1348
      if( !i1->is_Bool() ) continue;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1349
      BoolNode *bol = i1->as_Bool();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1350
      BoolTest b_test = bol->_test;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1351
      // Flip sense of test if exit condition is flipped
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1352
      if( flip )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1353
        b_test = b_test.negate();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1354
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1355
      // Get compare
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1356
      Node *cmp = bol->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1357
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1358
      // Look for trip_counter + offset vs limit
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1359
      Node *rc_exp = cmp->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1360
      Node *limit  = cmp->in(2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1361
      jint scale_con= 1;        // Assume trip counter not scaled
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1362
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1363
      Node *limit_c = get_ctrl(limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1364
      if( loop->is_member(get_loop(limit_c) ) ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1365
        // Compare might have operands swapped; commute them
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1366
        b_test = b_test.commute();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1367
        rc_exp = cmp->in(2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1368
        limit  = cmp->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1369
        limit_c = get_ctrl(limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1370
        if( loop->is_member(get_loop(limit_c) ) )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1371
          continue;             // Both inputs are loop varying; cannot RCE
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1372
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1373
      // Here we know 'limit' is loop invariant
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1374
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1375
      // 'limit' maybe pinned below the zero trip test (probably from a
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1376
      // previous round of rce), in which case, it can't be used in the
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1377
      // zero trip test expression which must occur before the zero test's if.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1378
      if( limit_c == ctrl ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1379
        continue;  // Don't rce this check but continue looking for other candidates.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1380
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1381
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1382
      // Check for scaled induction variable plus an offset
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1383
      Node *offset = NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1384
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1385
      if (!is_scaled_iv_plus_offset(rc_exp, trip_counter, &scale_con, &offset)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1386
        continue;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1387
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1388
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1389
      Node *offset_c = get_ctrl(offset);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1390
      if( loop->is_member( get_loop(offset_c) ) )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1391
        continue;               // Offset is not really loop invariant
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1392
      // Here we know 'offset' is loop invariant.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1393
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1394
      // As above for the 'limit', the 'offset' maybe pinned below the
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1395
      // zero trip test.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1396
      if( offset_c == ctrl ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1397
        continue; // Don't rce this check but continue looking for other candidates.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1398
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1399
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1400
      // At this point we have the expression as:
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1401
      //   scale_con * trip_counter + offset :: limit
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1402
      // where scale_con, offset and limit are loop invariant.  Trip_counter
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1403
      // monotonically increases by stride_con, a constant.  Both (or either)
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1404
      // stride_con and scale_con can be negative which will flip about the
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1405
      // sense of the test.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1406
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1407
      // Adjust pre and main loop limits to guard the correct iteration set
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1408
      if( cmp->Opcode() == Op_CmpU ) {// Unsigned compare is really 2 tests
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1409
        if( b_test._test == BoolTest::lt ) { // Range checks always use lt
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1410
          // The overflow limit: scale*I+offset < limit
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1411
          add_constraint( stride_con, scale_con, offset, limit, pre_ctrl, &pre_limit, &main_limit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1412
          // The underflow limit: 0 <= scale*I+offset.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1413
          // Some math yields: -scale*I-(offset+1) < 0
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1414
          Node *plus_one = new (C, 3) AddINode( offset, one );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1415
          register_new_node( plus_one, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1416
          Node *neg_offset = new (C, 3) SubINode( zero, plus_one );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1417
          register_new_node( neg_offset, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1418
          add_constraint( stride_con, -scale_con, neg_offset, zero, pre_ctrl, &pre_limit, &main_limit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1419
          if (!conditional_rc) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1420
            conditional_rc = !loop->dominates_backedge(iff);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1421
          }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1422
        } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1423
#ifndef PRODUCT
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1424
          if( PrintOpto )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1425
            tty->print_cr("missed RCE opportunity");
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1426
#endif
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1427
          continue;             // In release mode, ignore it
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1428
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1429
      } else {                  // Otherwise work on normal compares
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1430
        switch( b_test._test ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1431
        case BoolTest::ge:      // Convert X >= Y to -X <= -Y
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1432
          scale_con = -scale_con;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1433
          offset = new (C, 3) SubINode( zero, offset );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1434
          register_new_node( offset, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1435
          limit  = new (C, 3) SubINode( zero, limit  );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1436
          register_new_node( limit, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1437
          // Fall into LE case
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1438
        case BoolTest::le:      // Convert X <= Y to X < Y+1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1439
          limit = new (C, 3) AddINode( limit, one );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1440
          register_new_node( limit, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1441
          // Fall into LT case
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1442
        case BoolTest::lt:
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1443
          add_constraint( stride_con, scale_con, offset, limit, pre_ctrl, &pre_limit, &main_limit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1444
          if (!conditional_rc) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1445
            conditional_rc = !loop->dominates_backedge(iff);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1446
          }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1447
          break;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1448
        default:
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1449
#ifndef PRODUCT
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1450
          if( PrintOpto )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1451
            tty->print_cr("missed RCE opportunity");
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1452
#endif
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1453
          continue;             // Unhandled case
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1454
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1455
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1456
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1457
      // Kill the eliminated test
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1458
      C->set_major_progress();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1459
      Node *kill_con = _igvn.intcon( 1-flip );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1460
      set_ctrl(kill_con, C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1461
      _igvn.hash_delete(iff);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1462
      iff->set_req(1, kill_con);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1463
      _igvn._worklist.push(iff);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1464
      // Find surviving projection
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1465
      assert(iff->is_If(), "");
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1466
      ProjNode* dp = ((IfNode*)iff)->proj_out(1-flip);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1467
      // Find loads off the surviving projection; remove their control edge
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1468
      for (DUIterator_Fast imax, i = dp->fast_outs(imax); i < imax; i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1469
        Node* cd = dp->fast_out(i); // Control-dependent node
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1470
        if( cd->is_Load() ) {   // Loads can now float around in the loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1471
          _igvn.hash_delete(cd);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1472
          // Allow the load to float around in the loop, or before it
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1473
          // but NOT before the pre-loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1474
          cd->set_req(0, ctrl);   // ctrl, not NULL
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1475
          _igvn._worklist.push(cd);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1476
          --i;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1477
          --imax;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1478
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1479
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1480
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1481
    } // End of is IF
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1482
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1483
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1484
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1485
  // Update loop limits
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1486
  if (conditional_rc) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1487
    pre_limit = (stride_con > 0) ? (Node*)new (C,3) MinINode(pre_limit, orig_limit)
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1488
                                 : (Node*)new (C,3) MaxINode(pre_limit, orig_limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1489
    register_new_node(pre_limit, pre_ctrl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1490
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1491
  _igvn.hash_delete(pre_opaq);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1492
  pre_opaq->set_req(1, pre_limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1493
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1494
  // Note:: we are making the main loop limit no longer precise;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1495
  // need to round up based on stride.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1496
  if( stride_con != 1 && stride_con != -1 ) { // Cutout for common case
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1497
    // "Standard" round-up logic:  ([main_limit-init+(y-1)]/y)*y+init
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1498
    // Hopefully, compiler will optimize for powers of 2.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1499
    Node *ctrl = get_ctrl(main_limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1500
    Node *stride = cl->stride();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1501
    Node *init = cl->init_trip();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1502
    Node *span = new (C, 3) SubINode(main_limit,init);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1503
    register_new_node(span,ctrl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1504
    Node *rndup = _igvn.intcon(stride_con + ((stride_con>0)?-1:1));
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1505
    Node *add = new (C, 3) AddINode(span,rndup);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1506
    register_new_node(add,ctrl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1507
    Node *div = new (C, 3) DivINode(0,add,stride);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1508
    register_new_node(div,ctrl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1509
    Node *mul = new (C, 3) MulINode(div,stride);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1510
    register_new_node(mul,ctrl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1511
    Node *newlim = new (C, 3) AddINode(mul,init);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1512
    register_new_node(newlim,ctrl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1513
    main_limit = newlim;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1514
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1515
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1516
  Node *main_cle = cl->loopexit();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1517
  Node *main_bol = main_cle->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1518
  // Hacking loop bounds; need private copies of exit test
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1519
  if( main_bol->outcnt() > 1 ) {// BoolNode shared?
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1520
    _igvn.hash_delete(main_cle);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1521
    main_bol = main_bol->clone();// Clone a private BoolNode
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1522
    register_new_node( main_bol, main_cle->in(0) );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1523
    main_cle->set_req(1,main_bol);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1524
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1525
  Node *main_cmp = main_bol->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1526
  if( main_cmp->outcnt() > 1 ) { // CmpNode shared?
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1527
    _igvn.hash_delete(main_bol);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1528
    main_cmp = main_cmp->clone();// Clone a private CmpNode
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1529
    register_new_node( main_cmp, main_cle->in(0) );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1530
    main_bol->set_req(1,main_cmp);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1531
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1532
  // Hack the now-private loop bounds
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1533
  _igvn.hash_delete(main_cmp);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1534
  main_cmp->set_req(2, main_limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1535
  _igvn._worklist.push(main_cmp);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1536
  // The OpaqueNode is unshared by design
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1537
  _igvn.hash_delete(opqzm);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1538
  assert( opqzm->outcnt() == 1, "cannot hack shared node" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1539
  opqzm->set_req(1,main_limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1540
  _igvn._worklist.push(opqzm);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1541
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1542
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1543
//------------------------------DCE_loop_body----------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1544
// Remove simplistic dead code from loop body
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1545
void IdealLoopTree::DCE_loop_body() {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1546
  for( uint i = 0; i < _body.size(); i++ )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1547
    if( _body.at(i)->outcnt() == 0 )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1548
      _body.map( i--, _body.pop() );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1549
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1550
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1551
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1552
//------------------------------adjust_loop_exit_prob--------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1553
// Look for loop-exit tests with the 50/50 (or worse) guesses from the parsing stage.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1554
// Replace with a 1-in-10 exit guess.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1555
void IdealLoopTree::adjust_loop_exit_prob( PhaseIdealLoop *phase ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1556
  Node *test = tail();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1557
  while( test != _head ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1558
    uint top = test->Opcode();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1559
    if( top == Op_IfTrue || top == Op_IfFalse ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1560
      int test_con = ((ProjNode*)test)->_con;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1561
      assert(top == (uint)(test_con? Op_IfTrue: Op_IfFalse), "sanity");
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1562
      IfNode *iff = test->in(0)->as_If();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1563
      if( iff->outcnt() == 2 ) {        // Ignore dead tests
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1564
        Node *bol = iff->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1565
        if( bol && bol->req() > 1 && bol->in(1) &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1566
            ((bol->in(1)->Opcode() == Op_StorePConditional ) ||
1500
bea9a90f3e8f 6462850: generate biased locking code in C2 ideal graph
kvn
parents: 1433
diff changeset
  1567
             (bol->in(1)->Opcode() == Op_StoreIConditional ) ||
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1568
             (bol->in(1)->Opcode() == Op_StoreLConditional ) ||
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1569
             (bol->in(1)->Opcode() == Op_CompareAndSwapI ) ||
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1570
             (bol->in(1)->Opcode() == Op_CompareAndSwapL ) ||
360
21d113ecbf6a 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 212
diff changeset
  1571
             (bol->in(1)->Opcode() == Op_CompareAndSwapP ) ||
21d113ecbf6a 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 212
diff changeset
  1572
             (bol->in(1)->Opcode() == Op_CompareAndSwapN )))
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1573
          return;               // Allocation loops RARELY take backedge
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1574
        // Find the OTHER exit path from the IF
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1575
        Node* ex = iff->proj_out(1-test_con);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1576
        float p = iff->_prob;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1577
        if( !phase->is_member( this, ex ) && iff->_fcnt == COUNT_UNKNOWN ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1578
          if( top == Op_IfTrue ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1579
            if( p < (PROB_FAIR + PROB_UNLIKELY_MAG(3))) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1580
              iff->_prob = PROB_STATIC_FREQUENT;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1581
            }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1582
          } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1583
            if( p > (PROB_FAIR - PROB_UNLIKELY_MAG(3))) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1584
              iff->_prob = PROB_STATIC_INFREQUENT;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1585
            }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1586
          }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1587
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1588
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1589
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1590
    test = phase->idom(test);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1591
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1592
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1593
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1594
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1595
//------------------------------policy_do_remove_empty_loop--------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1596
// Micro-benchmark spamming.  Policy is to always remove empty loops.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1597
// The 'DO' part is to replace the trip counter with the value it will
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1598
// have on the last iteration.  This will break the loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1599
bool IdealLoopTree::policy_do_remove_empty_loop( PhaseIdealLoop *phase ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1600
  // Minimum size must be empty loop
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1601
  if (_body.size() > 7/*number of nodes in an empty loop*/)
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1602
    return false;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1603
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1604
  if (!_head->is_CountedLoop())
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1605
    return false;     // Dead loop
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1606
  CountedLoopNode *cl = _head->as_CountedLoop();
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1607
  if (!cl->loopexit())
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1608
    return false; // Malformed loop
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1609
  if (!phase->is_member(this, phase->get_ctrl(cl->loopexit()->in(CountedLoopEndNode::TestValue))))
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1610
    return false;             // Infinite loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1611
#ifndef PRODUCT
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1612
  if (PrintOpto) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1613
    tty->print("Removing empty loop");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1614
    this->dump_head();
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1615
  } else if (TraceLoopOpts) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1616
    tty->print("Empty        ");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1617
    this->dump_head();
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1618
  }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1619
#endif
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1620
#ifdef ASSERT
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1621
  // Ensure only one phi which is the iv.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1622
  Node* iv = NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1623
  for (DUIterator_Fast imax, i = cl->fast_outs(imax); i < imax; i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1624
    Node* n = cl->fast_out(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1625
    if (n->Opcode() == Op_Phi) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1626
      assert(iv == NULL, "Too many phis" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1627
      iv = n;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1628
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1629
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1630
  assert(iv == cl->phi(), "Wrong phi" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1631
#endif
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1632
  // Replace the phi at loop head with the final value of the last
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1633
  // iteration.  Then the CountedLoopEnd will collapse (backedge never
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1634
  // taken) and all loop-invariant uses of the exit values will be correct.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1635
  Node *phi = cl->phi();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1636
  Node *final = new (phase->C, 3) SubINode( cl->limit(), cl->stride() );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1637
  phase->register_new_node(final,cl->in(LoopNode::EntryControl));
5901
c046f8e9c52b 6677629: PhaseIterGVN::subsume_node() should call hash_delete() and add_users_to_worklist()
kvn
parents: 5547
diff changeset
  1638
  phase->_igvn.replace_node(phi,final);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1639
  phase->C->set_major_progress();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1640
  return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1641
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1642
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1643
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1644
//=============================================================================
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1645
//------------------------------iteration_split_impl---------------------------
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1646
bool IdealLoopTree::iteration_split_impl( PhaseIdealLoop *phase, Node_List &old_new ) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1647
  // Check and remove empty loops (spam micro-benchmarks)
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1648
  if( policy_do_remove_empty_loop(phase) )
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1649
    return true;  // Here we removed an empty loop
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1650
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1651
  bool should_peel = policy_peeling(phase); // Should we peel?
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1652
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1653
  bool should_unswitch = policy_unswitching(phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1654
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1655
  // Non-counted loops may be peeled; exactly 1 iteration is peeled.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1656
  // This removes loop-invariant tests (usually null checks).
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1657
  if( !_head->is_CountedLoop() ) { // Non-counted loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1658
    if (PartialPeelLoop && phase->partial_peel(this, old_new)) {
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1659
      // Partial peel succeeded so terminate this round of loop opts
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1660
      return false;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1661
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1662
    if( should_peel ) {            // Should we peel?
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1663
#ifndef PRODUCT
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1664
      if (PrintOpto) tty->print_cr("should_peel");
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1665
#endif
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1666
      phase->do_peeling(this,old_new);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1667
    } else if( should_unswitch ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1668
      phase->do_unswitching(this, old_new);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1669
    }
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1670
    return true;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1671
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1672
  CountedLoopNode *cl = _head->as_CountedLoop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1673
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1674
  if( !cl->loopexit() ) return true; // Ignore various kinds of broken loops
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1675
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1676
  // Do nothing special to pre- and post- loops
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1677
  if( cl->is_pre_loop() || cl->is_post_loop() ) return true;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1678
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1679
  // Compute loop trip count from profile data
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1680
  compute_profile_trip_cnt(phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1681
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1682
  // Before attempting fancy unrolling, RCE or alignment, see if we want
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1683
  // to completely unroll this loop or do loop unswitching.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1684
  if( cl->is_normal_loop() ) {
2876
b8ebc85d0b47 6837146: Should perform unswitch before maximally unroll in loop transformation
cfang
parents: 2131
diff changeset
  1685
    if (should_unswitch) {
b8ebc85d0b47 6837146: Should perform unswitch before maximally unroll in loop transformation
cfang
parents: 2131
diff changeset
  1686
      phase->do_unswitching(this, old_new);
b8ebc85d0b47 6837146: Should perform unswitch before maximally unroll in loop transformation
cfang
parents: 2131
diff changeset
  1687
      return true;
b8ebc85d0b47 6837146: Should perform unswitch before maximally unroll in loop transformation
cfang
parents: 2131
diff changeset
  1688
    }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1689
    bool should_maximally_unroll =  policy_maximally_unroll(phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1690
    if( should_maximally_unroll ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1691
      // Here we did some unrolling and peeling.  Eventually we will
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1692
      // completely unroll this loop and it will no longer be a loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1693
      phase->do_maximally_unroll(this,old_new);
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1694
      return true;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1695
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1696
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1697
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1698
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1699
  // Counted loops may be peeled, may need some iterations run up
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1700
  // front for RCE, and may want to align loop refs to a cache
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1701
  // line.  Thus we clone a full loop up front whose trip count is
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1702
  // at least 1 (if peeling), but may be several more.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1703
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1704
  // The main loop will start cache-line aligned with at least 1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1705
  // iteration of the unrolled body (zero-trip test required) and
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1706
  // will have some range checks removed.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1707
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1708
  // A post-loop will finish any odd iterations (leftover after
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1709
  // unrolling), plus any needed for RCE purposes.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1710
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1711
  bool should_unroll = policy_unroll(phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1712
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1713
  bool should_rce = policy_range_check(phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1714
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1715
  bool should_align = policy_align(phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1716
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1717
  // If not RCE'ing (iteration splitting) or Aligning, then we do not
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1718
  // need a pre-loop.  We may still need to peel an initial iteration but
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1719
  // we will not be needing an unknown number of pre-iterations.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1720
  //
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1721
  // Basically, if may_rce_align reports FALSE first time through,
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1722
  // we will not be able to later do RCE or Aligning on this loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1723
  bool may_rce_align = !policy_peel_only(phase) || should_rce || should_align;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1724
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1725
  // If we have any of these conditions (RCE, alignment, unrolling) met, then
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1726
  // we switch to the pre-/main-/post-loop model.  This model also covers
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1727
  // peeling.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1728
  if( should_rce || should_align || should_unroll ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1729
    if( cl->is_normal_loop() )  // Convert to 'pre/main/post' loops
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1730
      phase->insert_pre_post_loops(this,old_new, !may_rce_align);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1731
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1732
    // Adjust the pre- and main-loop limits to let the pre and post loops run
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1733
    // with full checks, but the main-loop with no checks.  Remove said
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1734
    // checks from the main body.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1735
    if( should_rce )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1736
      phase->do_range_check(this,old_new);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1737
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1738
    // Double loop body for unrolling.  Adjust the minimum-trip test (will do
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1739
    // twice as many iterations as before) and the main body limit (only do
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1740
    // an even number of trips).  If we are peeling, we might enable some RCE
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1741
    // and we'd rather unroll the post-RCE'd loop SO... do not unroll if
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1742
    // peeling.
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1743
      if( should_unroll && !should_peel )
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1744
        phase->do_unroll(this,old_new, true);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1745
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1746
    // Adjust the pre-loop limits to align the main body
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1747
    // iterations.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1748
    if( should_align )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1749
      Unimplemented();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1750
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1751
  } else {                      // Else we have an unchanged counted loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1752
    if( should_peel )           // Might want to peel but do nothing else
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1753
      phase->do_peeling(this,old_new);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1754
  }
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1755
  return true;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1756
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1757
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1758
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1759
//=============================================================================
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1760
//------------------------------iteration_split--------------------------------
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1761
bool IdealLoopTree::iteration_split( PhaseIdealLoop *phase, Node_List &old_new ) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1762
  // Recursively iteration split nested loops
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1763
  if (_child && !_child->iteration_split(phase, old_new))
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1764
    return false;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1765
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1766
  // Clean out prior deadwood
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1767
  DCE_loop_body();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1768
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1769
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1770
  // Look for loop-exit tests with my 50/50 guesses from the Parsing stage.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1771
  // Replace with a 1-in-10 exit guess.
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1772
  if (_parent /*not the root loop*/ &&
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1773
      !_irreducible &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1774
      // Also ignore the occasional dead backedge
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1775
      !tail()->is_top()) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1776
    adjust_loop_exit_prob(phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1777
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1778
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1779
  // Gate unrolling, RCE and peeling efforts.
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1780
  if (!_child &&                // If not an inner loop, do not split
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1781
      !_irreducible &&
212
cd4963e67949 6667612: (Escape Analysis) disable loop cloning if it has a scalar replaceable allocation
kvn
parents: 1
diff changeset
  1782
      _allow_optimizations &&
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1783
      !tail()->is_top()) {     // Also ignore the occasional dead backedge
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1784
    if (!_has_call) {
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1785
        if (!iteration_split_impl(phase, old_new)) {
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1786
          return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1787
        }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1788
    } else if (policy_unswitching(phase)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1789
      phase->do_unswitching(this, old_new);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1790
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1791
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1792
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1793
  // Minor offset re-organization to remove loop-fallout uses of
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1794
  // trip counter when there was no major reshaping.
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1795
  phase->reorg_offsets(this);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1796
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1797
  if (_next && !_next->iteration_split(phase, old_new))
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1798
    return false;
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1799
  return true;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1800
}
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1801
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1802
//-------------------------------is_uncommon_trap_proj----------------------------
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1803
// Return true if proj is the form of "proj->[region->..]call_uct"
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1804
bool PhaseIdealLoop::is_uncommon_trap_proj(ProjNode* proj, Deoptimization::DeoptReason reason) {
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1805
  int path_limit = 10;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1806
  assert(proj, "invalid argument");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1807
  Node* out = proj;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1808
  for (int ct = 0; ct < path_limit; ct++) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1809
    out = out->unique_ctrl_out();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1810
    if (out == NULL || out->is_Root() || out->is_Start())
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1811
      return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1812
    if (out->is_CallStaticJava()) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1813
      int req = out->as_CallStaticJava()->uncommon_trap_request();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1814
      if (req != 0) {
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1815
        Deoptimization::DeoptReason trap_reason = Deoptimization::trap_request_reason(req);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1816
        if (trap_reason == reason || reason == Deoptimization::Reason_none) {
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1817
           return true;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1818
        }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1819
      }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1820
      return false; // don't do further after call
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1821
    }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1822
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1823
  return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1824
}
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1825
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1826
//-------------------------------is_uncommon_trap_if_pattern-------------------------
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1827
// Return true  for "if(test)-> proj -> ...
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1828
//                          |
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1829
//                          V
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1830
//                      other_proj->[region->..]call_uct"
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1831
//
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1832
// "must_reason_predicate" means the uct reason must be Reason_predicate
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1833
bool PhaseIdealLoop::is_uncommon_trap_if_pattern(ProjNode *proj, Deoptimization::DeoptReason reason) {
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1834
  Node *in0 = proj->in(0);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1835
  if (!in0->is_If()) return false;
5031
80fd89e1bd57 6930116: loop predication code does not handle If nodes with only one projection
kvn
parents: 5026
diff changeset
  1836
  // Variation of a dead If node.
80fd89e1bd57 6930116: loop predication code does not handle If nodes with only one projection
kvn
parents: 5026
diff changeset
  1837
  if (in0->outcnt() < 2)  return false;
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1838
  IfNode* iff = in0->as_If();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1839
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1840
  // we need "If(Conv2B(Opaque1(...)))" pattern for reason_predicate
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1841
  if (reason != Deoptimization::Reason_none) {
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1842
    if (iff->in(1)->Opcode() != Op_Conv2B ||
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1843
       iff->in(1)->in(1)->Opcode() != Op_Opaque1) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1844
      return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1845
    }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1846
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1847
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1848
  ProjNode* other_proj = iff->proj_out(1-proj->_con)->as_Proj();
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1849
  return is_uncommon_trap_proj(other_proj, reason);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1850
}
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1851
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1852
//-------------------------------register_control-------------------------
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1853
void PhaseIdealLoop::register_control(Node* n, IdealLoopTree *loop, Node* pred) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1854
  assert(n->is_CFG(), "must be control node");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1855
  _igvn.register_new_node_with_optimizer(n);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1856
  loop->_body.push(n);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1857
  set_loop(n, loop);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1858
  // When called from beautify_loops() idom is not constructed yet.
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1859
  if (_idom != NULL) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1860
    set_idom(n, pred, dom_depth(pred));
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1861
  }
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1862
}
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1863
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1864
//------------------------------create_new_if_for_predicate------------------------
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1865
// create a new if above the uct_if_pattern for the predicate to be promoted.
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1866
//
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1867
//          before                                after
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1868
//        ----------                           ----------
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1869
//           ctrl                                 ctrl
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1870
//            |                                     |
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1871
//            |                                     |
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1872
//            v                                     v
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1873
//           iff                                 new_iff
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1874
//          /    \                                /      \
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1875
//         /      \                              /        \
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1876
//        v        v                            v          v
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1877
//  uncommon_proj cont_proj                   if_uct     if_cont
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1878
// \      |        |                           |          |
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1879
//  \     |        |                           |          |
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1880
//   v    v        v                           |          v
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1881
//     rgn       loop                          |         iff
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1882
//      |                                      |        /     \
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1883
//      |                                      |       /       \
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1884
//      v                                      |      v         v
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1885
// uncommon_trap                               | uncommon_proj cont_proj
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1886
//                                           \  \    |           |
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1887
//                                            \  \   |           |
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1888
//                                             v  v  v           v
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1889
//                                               rgn           loop
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1890
//                                                |
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1891
//                                                |
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1892
//                                                v
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1893
//                                           uncommon_trap
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1894
//
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1895
//
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1896
// We will create a region to guard the uct call if there is no one there.
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1897
// The true projecttion (if_cont) of the new_iff is returned.
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1898
// This code is also used to clone predicates to clonned loops.
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1899
ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry,
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1900
                                                      Deoptimization::DeoptReason reason) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1901
  assert(is_uncommon_trap_if_pattern(cont_proj, reason), "must be a uct if pattern!");
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1902
  IfNode* iff = cont_proj->in(0)->as_If();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1903
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1904
  ProjNode *uncommon_proj = iff->proj_out(1 - cont_proj->_con);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1905
  Node     *rgn   = uncommon_proj->unique_ctrl_out();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1906
  assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1907
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1908
  if (!rgn->is_Region()) { // create a region to guard the call
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1909
    assert(rgn->is_Call(), "must be call uct");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1910
    CallNode* call = rgn->as_Call();
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1911
    IdealLoopTree* loop = get_loop(call);
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1912
    rgn = new (C, 1) RegionNode(1);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1913
    rgn->add_req(uncommon_proj);
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1914
    register_control(rgn, loop, uncommon_proj);
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1915
    _igvn.hash_delete(call);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1916
    call->set_req(0, rgn);
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1917
    // When called from beautify_loops() idom is not constructed yet.
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1918
    if (_idom != NULL) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1919
      set_idom(call, rgn, dom_depth(rgn));
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1920
    }
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1921
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1922
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1923
  Node* entry = iff->in(0);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1924
  if (new_entry != NULL) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1925
    // Clonning the predicate to new location.
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1926
    entry = new_entry;
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1927
  }
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1928
  // Create new_iff
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1929
  IdealLoopTree* lp = get_loop(entry);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1930
  IfNode *new_iff = new (C, 2) IfNode(entry, NULL, iff->_prob, iff->_fcnt);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1931
  register_control(new_iff, lp, entry);
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1932
  Node *if_cont = new (C, 1) IfTrueNode(new_iff);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1933
  Node *if_uct  = new (C, 1) IfFalseNode(new_iff);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1934
  if (cont_proj->is_IfFalse()) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1935
    // Swap
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1936
    Node* tmp = if_uct; if_uct = if_cont; if_cont = tmp;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1937
  }
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1938
  register_control(if_cont, lp, new_iff);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1939
  register_control(if_uct, get_loop(rgn), new_iff);
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1940
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1941
  // if_uct to rgn
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1942
  _igvn.hash_delete(rgn);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1943
  rgn->add_req(if_uct);
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1944
  // When called from beautify_loops() idom is not constructed yet.
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1945
  if (_idom != NULL) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1946
    Node* ridom = idom(rgn);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1947
    Node* nrdom = dom_lca(ridom, new_iff);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1948
    set_idom(rgn, nrdom, dom_depth(rgn));
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1949
  }
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1950
  // rgn must have no phis
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1951
  assert(!rgn->as_Region()->has_phi(), "region must have no phis");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1952
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1953
  if (new_entry == NULL) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1954
    // Attach if_cont to iff
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1955
    _igvn.hash_delete(iff);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1956
    iff->set_req(0, if_cont);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1957
    if (_idom != NULL) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1958
      set_idom(iff, if_cont, dom_depth(iff));
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1959
    }
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1960
  }
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1961
  return if_cont->as_Proj();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1962
}
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1963
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1964
//--------------------------find_predicate_insertion_point-------------------
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1965
// Find a good location to insert a predicate
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1966
ProjNode* PhaseIdealLoop::find_predicate_insertion_point(Node* start_c, Deoptimization::DeoptReason reason) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1967
  if (start_c == NULL || !start_c->is_Proj())
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1968
    return NULL;
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1969
  if (is_uncommon_trap_if_pattern(start_c->as_Proj(), reason)) {
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1970
    return start_c->as_Proj();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1971
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1972
  return NULL;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1973
}
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1974
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1975
//--------------------------find_predicate------------------------------------
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1976
// Find a predicate
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1977
Node* PhaseIdealLoop::find_predicate(Node* entry) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1978
  Node* predicate = NULL;
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1979
  if (UseLoopPredicate) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1980
    predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1981
    if (predicate != NULL) { // right pattern that can be used by loop predication
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1982
      assert(entry->in(0)->in(1)->in(1)->Opcode()==Op_Opaque1, "must be");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1983
      return entry;
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1984
    }
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1985
  }
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1986
  return NULL;
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1987
}
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1988
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1989
//------------------------------Invariance-----------------------------------
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1990
// Helper class for loop_predication_impl to compute invariance on the fly and
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1991
// clone invariants.
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1992
class Invariance : public StackObj {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1993
  VectorSet _visited, _invariant;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1994
  Node_Stack _stack;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1995
  VectorSet _clone_visited;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1996
  Node_List _old_new; // map of old to new (clone)
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1997
  IdealLoopTree* _lpt;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1998
  PhaseIdealLoop* _phase;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1999
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2000
  // Helper function to set up the invariance for invariance computation
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2001
  // If n is a known invariant, set up directly. Otherwise, look up the
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2002
  // the possibility to push n onto the stack for further processing.
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2003
  void visit(Node* use, Node* n) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2004
    if (_lpt->is_invariant(n)) { // known invariant
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2005
      _invariant.set(n->_idx);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2006
    } else if (!n->is_CFG()) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2007
      Node *n_ctrl = _phase->ctrl_or_self(n);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2008
      Node *u_ctrl = _phase->ctrl_or_self(use); // self if use is a CFG
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2009
      if (_phase->is_dominator(n_ctrl, u_ctrl)) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2010
        _stack.push(n, n->in(0) == NULL ? 1 : 0);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2011
      }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2012
    }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2013
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2014
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2015
  // Compute invariance for "the_node" and (possibly) all its inputs recursively
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2016
  // on the fly
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2017
  void compute_invariance(Node* n) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2018
    assert(_visited.test(n->_idx), "must be");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2019
    visit(n, n);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2020
    while (_stack.is_nonempty()) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2021
      Node*  n = _stack.node();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2022
      uint idx = _stack.index();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2023
      if (idx == n->req()) { // all inputs are processed
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2024
        _stack.pop();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2025
        // n is invariant if it's inputs are all invariant
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2026
        bool all_inputs_invariant = true;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2027
        for (uint i = 0; i < n->req(); i++) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2028
          Node* in = n->in(i);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2029
          if (in == NULL) continue;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2030
          assert(_visited.test(in->_idx), "must have visited input");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2031
          if (!_invariant.test(in->_idx)) { // bad guy
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2032
            all_inputs_invariant = false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2033
            break;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2034
          }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2035
        }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2036
        if (all_inputs_invariant) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2037
          _invariant.set(n->_idx); // I am a invariant too
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2038
        }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2039
      } else { // process next input
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2040
        _stack.set_index(idx + 1);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2041
        Node* m = n->in(idx);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2042
        if (m != NULL && !_visited.test_set(m->_idx)) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2043
          visit(n, m);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2044
        }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2045
      }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2046
    }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2047
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2048
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2049
  // Helper function to set up _old_new map for clone_nodes.
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2050
  // If n is a known invariant, set up directly ("clone" of n == n).
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2051
  // Otherwise, push n onto the stack for real cloning.
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2052
  void clone_visit(Node* n) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2053
    assert(_invariant.test(n->_idx), "must be invariant");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2054
    if (_lpt->is_invariant(n)) { // known invariant
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2055
      _old_new.map(n->_idx, n);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2056
    } else{ // to be cloned
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2057
      assert (!n->is_CFG(), "should not see CFG here");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2058
      _stack.push(n, n->in(0) == NULL ? 1 : 0);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2059
    }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2060
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2061
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2062
  // Clone "n" and (possibly) all its inputs recursively
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2063
  void clone_nodes(Node* n, Node* ctrl) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2064
    clone_visit(n);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2065
    while (_stack.is_nonempty()) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2066
      Node*  n = _stack.node();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2067
      uint idx = _stack.index();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2068
      if (idx == n->req()) { // all inputs processed, clone n!
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2069
        _stack.pop();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2070
        // clone invariant node
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2071
        Node* n_cl = n->clone();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2072
        _old_new.map(n->_idx, n_cl);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2073
        _phase->register_new_node(n_cl, ctrl);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2074
        for (uint i = 0; i < n->req(); i++) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2075
          Node* in = n_cl->in(i);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2076
          if (in == NULL) continue;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2077
          n_cl->set_req(i, _old_new[in->_idx]);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2078
        }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2079
      } else { // process next input
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2080
        _stack.set_index(idx + 1);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2081
        Node* m = n->in(idx);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2082
        if (m != NULL && !_clone_visited.test_set(m->_idx)) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2083
          clone_visit(m); // visit the input
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2084
        }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2085
      }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2086
    }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2087
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2088
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2089
 public:
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2090
  Invariance(Arena* area, IdealLoopTree* lpt) :
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2091
    _lpt(lpt), _phase(lpt->_phase),
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2092
    _visited(area), _invariant(area), _stack(area, 10 /* guess */),
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2093
    _clone_visited(area), _old_new(area)
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2094
  {}
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2095
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2096
  // Map old to n for invariance computation and clone
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2097
  void map_ctrl(Node* old, Node* n) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2098
    assert(old->is_CFG() && n->is_CFG(), "must be");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2099
    _old_new.map(old->_idx, n); // "clone" of old is n
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2100
    _invariant.set(old->_idx);  // old is invariant
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2101
    _clone_visited.set(old->_idx);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2102
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2103
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2104
  // Driver function to compute invariance
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2105
  bool is_invariant(Node* n) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2106
    if (!_visited.test_set(n->_idx))
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2107
      compute_invariance(n);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2108
    return (_invariant.test(n->_idx) != 0);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2109
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2110
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2111
  // Driver function to clone invariant
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2112
  Node* clone(Node* n, Node* ctrl) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2113
    assert(ctrl->is_CFG(), "must be");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2114
    assert(_invariant.test(n->_idx), "must be an invariant");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2115
    if (!_clone_visited.test(n->_idx))
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2116
      clone_nodes(n, ctrl);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2117
    return _old_new[n->_idx];
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2118
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2119
};
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2120
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2121
//------------------------------is_range_check_if -----------------------------------
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2122
// Returns true if the predicate of iff is in "scale*iv + offset u< load_range(ptr)" format
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2123
// Note: this function is particularly designed for loop predication. We require load_range
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2124
//       and offset to be loop invariant computed on the fly by "invar"
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2125
bool IdealLoopTree::is_range_check_if(IfNode *iff, PhaseIdealLoop *phase, Invariance& invar) const {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2126
  if (!is_loop_exit(iff)) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2127
    return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2128
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2129
  if (!iff->in(1)->is_Bool()) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2130
    return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2131
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2132
  const BoolNode *bol = iff->in(1)->as_Bool();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2133
  if (bol->_test._test != BoolTest::lt) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2134
    return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2135
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2136
  if (!bol->in(1)->is_Cmp()) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2137
    return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2138
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2139
  const CmpNode *cmp = bol->in(1)->as_Cmp();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2140
  if (cmp->Opcode() != Op_CmpU ) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2141
    return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2142
  }
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2143
  Node* range = cmp->in(2);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2144
  if (range->Opcode() != Op_LoadRange) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2145
    const TypeInt* tint = phase->_igvn.type(range)->isa_int();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2146
    if (!OptimizeFill || tint == NULL || tint->empty() || tint->_lo < 0) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2147
      // Allow predication on positive values that aren't LoadRanges.
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2148
      // This allows optimization of loops where the length of the
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2149
      // array is a known value and doesn't need to be loaded back
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2150
      // from the array.
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2151
      return false;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2152
    }
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2153
  }
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2154
  if (!invar.is_invariant(range)) {
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2155
    return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2156
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2157
  Node *iv     = _head->as_CountedLoop()->phi();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2158
  int   scale  = 0;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2159
  Node *offset = NULL;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2160
  if (!phase->is_scaled_iv_plus_offset(cmp->in(1), iv, &scale, &offset)) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2161
    return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2162
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2163
  if(offset && !invar.is_invariant(offset)) { // offset must be invariant
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2164
    return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2165
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2166
  return true;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2167
}
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2168
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2169
//------------------------------rc_predicate-----------------------------------
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2170
// Create a range check predicate
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2171
//
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2172
// for (i = init; i < limit; i += stride) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2173
//    a[scale*i+offset]
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2174
// }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2175
//
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2176
// Compute max(scale*i + offset) for init <= i < limit and build the predicate
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2177
// as "max(scale*i + offset) u< a.length".
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2178
//
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2179
// There are two cases for max(scale*i + offset):
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2180
// (1) stride*scale > 0
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2181
//   max(scale*i + offset) = scale*(limit-stride) + offset
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2182
// (2) stride*scale < 0
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2183
//   max(scale*i + offset) = scale*init + offset
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2184
BoolNode* PhaseIdealLoop::rc_predicate(Node* ctrl,
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2185
                                       int scale, Node* offset,
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2186
                                       Node* init, Node* limit, Node* stride,
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2187
                                       Node* range, bool upper) {
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2188
  DEBUG_ONLY(ttyLocker ttyl);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2189
  if (TraceLoopPredicate) tty->print("rc_predicate ");
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2190
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2191
  Node* max_idx_expr  = init;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2192
  int stride_con = stride->get_int();
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2193
  if ((stride_con > 0) == (scale > 0) == upper) {
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2194
    max_idx_expr = new (C, 3) SubINode(limit, stride);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2195
    register_new_node(max_idx_expr, ctrl);
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2196
    if (TraceLoopPredicate) tty->print("(limit - stride) ");
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2197
  } else {
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2198
    if (TraceLoopPredicate) tty->print("init ");
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2199
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2200
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2201
  if (scale != 1) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2202
    ConNode* con_scale = _igvn.intcon(scale);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2203
    max_idx_expr = new (C, 3) MulINode(max_idx_expr, con_scale);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2204
    register_new_node(max_idx_expr, ctrl);
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2205
    if (TraceLoopPredicate) tty->print("* %d ", scale);
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2206
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2207
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2208
  if (offset && (!offset->is_Con() || offset->get_int() != 0)){
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2209
    max_idx_expr = new (C, 3) AddINode(max_idx_expr, offset);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2210
    register_new_node(max_idx_expr, ctrl);
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2211
    if (TraceLoopPredicate)
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2212
      if (offset->is_Con()) tty->print("+ %d ", offset->get_int());
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2213
      else tty->print("+ offset ");
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2214
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2215
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2216
  CmpUNode* cmp = new (C, 3) CmpUNode(max_idx_expr, range);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2217
  register_new_node(cmp, ctrl);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2218
  BoolNode* bol = new (C, 2) BoolNode(cmp, BoolTest::lt);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2219
  register_new_node(bol, ctrl);
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2220
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2221
  if (TraceLoopPredicate) tty->print_cr("<u range");
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2222
  return bol;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2223
}
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2224
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2225
//------------------------------ loop_predication_impl--------------------------
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2226
// Insert loop predicates for null checks and range checks
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2227
bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2228
  if (!UseLoopPredicate) return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2229
5026
51f0e7aa2245 6927049: assert(is_Loop(),"invalid node class")
never
parents: 4643
diff changeset
  2230
  if (!loop->_head->is_Loop()) {
51f0e7aa2245 6927049: assert(is_Loop(),"invalid node class")
never
parents: 4643
diff changeset
  2231
    // Could be a simple region when irreducible loops are present.
51f0e7aa2245 6927049: assert(is_Loop(),"invalid node class")
never
parents: 4643
diff changeset
  2232
    return false;
51f0e7aa2245 6927049: assert(is_Loop(),"invalid node class")
never
parents: 4643
diff changeset
  2233
  }
51f0e7aa2245 6927049: assert(is_Loop(),"invalid node class")
never
parents: 4643
diff changeset
  2234
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2235
  if (loop->_head->unique_ctrl_out()->Opcode() == Op_NeverBranch) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2236
    // do nothing for infinite loops
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2237
    return false;
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2238
  }
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2239
5026
51f0e7aa2245 6927049: assert(is_Loop(),"invalid node class")
never
parents: 4643
diff changeset
  2240
  CountedLoopNode *cl = NULL;
51f0e7aa2245 6927049: assert(is_Loop(),"invalid node class")
never
parents: 4643
diff changeset
  2241
  if (loop->_head->is_CountedLoop()) {
51f0e7aa2245 6927049: assert(is_Loop(),"invalid node class")
never
parents: 4643
diff changeset
  2242
    cl = loop->_head->as_CountedLoop();
51f0e7aa2245 6927049: assert(is_Loop(),"invalid node class")
never
parents: 4643
diff changeset
  2243
    // do nothing for iteration-splitted loops
51f0e7aa2245 6927049: assert(is_Loop(),"invalid node class")
never
parents: 4643
diff changeset
  2244
    if (!cl->is_normal_loop()) return false;
51f0e7aa2245 6927049: assert(is_Loop(),"invalid node class")
never
parents: 4643
diff changeset
  2245
  }
51f0e7aa2245 6927049: assert(is_Loop(),"invalid node class")
never
parents: 4643
diff changeset
  2246
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2247
  LoopNode *lpn  = loop->_head->as_Loop();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2248
  Node* entry = lpn->in(LoopNode::EntryControl);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2249
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2250
  ProjNode *predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2251
  if (!predicate_proj) {
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2252
#ifndef PRODUCT
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2253
    if (TraceLoopPredicate) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2254
      tty->print("missing predicate:");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2255
      loop->dump_head();
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2256
      lpn->dump(1);
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2257
    }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2258
#endif
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2259
    return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2260
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2261
  ConNode* zero = _igvn.intcon(0);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2262
  set_ctrl(zero, C->root());
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2263
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2264
  ResourceArea *area = Thread::current()->resource_area();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2265
  Invariance invar(area, loop);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2266
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2267
  // Create list of if-projs such that a newer proj dominates all older
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2268
  // projs in the list, and they all dominate loop->tail()
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2269
  Node_List if_proj_list(area);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2270
  LoopNode *head  = loop->_head->as_Loop();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2271
  Node *current_proj = loop->tail(); //start from tail
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2272
  while ( current_proj != head ) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2273
    if (loop == get_loop(current_proj) && // still in the loop ?
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2274
        current_proj->is_Proj()        && // is a projection  ?
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2275
        current_proj->in(0)->Opcode() == Op_If) { // is a if projection ?
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2276
      if_proj_list.push(current_proj);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2277
    }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2278
    current_proj = idom(current_proj);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2279
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2280
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2281
  bool hoisted = false; // true if at least one proj is promoted
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2282
  while (if_proj_list.size() > 0) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2283
    // Following are changed to nonnull when a predicate can be hoisted
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2284
    ProjNode* new_predicate_proj = NULL;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2285
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2286
    ProjNode* proj = if_proj_list.pop()->as_Proj();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2287
    IfNode*   iff  = proj->in(0)->as_If();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2288
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2289
    if (!is_uncommon_trap_if_pattern(proj, Deoptimization::Reason_none)) {
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2290
      if (loop->is_loop_exit(iff)) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2291
        // stop processing the remaining projs in the list because the execution of them
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2292
        // depends on the condition of "iff" (iff->in(1)).
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2293
        break;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2294
      } else {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2295
        // Both arms are inside the loop. There are two cases:
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2296
        // (1) there is one backward branch. In this case, any remaining proj
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2297
        //     in the if_proj list post-dominates "iff". So, the condition of "iff"
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2298
        //     does not determine the execution the remining projs directly, and we
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2299
        //     can safely continue.
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2300
        // (2) both arms are forwarded, i.e. a diamond shape. In this case, "proj"
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2301
        //     does not dominate loop->tail(), so it can not be in the if_proj list.
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2302
        continue;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2303
      }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2304
    }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2305
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2306
    Node*     test = iff->in(1);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2307
    if (!test->is_Bool()){ //Conv2B, ...
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2308
      continue;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2309
    }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2310
    BoolNode* bol = test->as_Bool();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2311
    if (invar.is_invariant(bol)) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2312
      // Invariant test
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2313
      new_predicate_proj = create_new_if_for_predicate(predicate_proj, NULL,
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2314
                                                       Deoptimization::Reason_predicate);
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2315
      Node* ctrl = new_predicate_proj->in(0)->as_If()->in(0);
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2316
      BoolNode* new_predicate_bol = invar.clone(bol, ctrl)->as_Bool();
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2317
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2318
      // Negate test if necessary
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2319
      bool negated = false;
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2320
      if (proj->_con != predicate_proj->_con) {
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2321
        new_predicate_bol = new (C, 2) BoolNode(new_predicate_bol->in(1), new_predicate_bol->_test.negate());
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2322
        register_new_node(new_predicate_bol, ctrl);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2323
        negated = true;
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2324
      }
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2325
      IfNode* new_predicate_iff = new_predicate_proj->in(0)->as_If();
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2326
      _igvn.hash_delete(new_predicate_iff);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2327
      new_predicate_iff->set_req(1, new_predicate_bol);
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2328
#ifndef PRODUCT
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2329
      if (TraceLoopPredicate) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2330
        tty->print("Predicate invariant if%s: %d ", negated ? " negated" : "", new_predicate_iff->_idx);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2331
        loop->dump_head();
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2332
      } else if (TraceLoopOpts) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2333
        tty->print("Predicate IC ");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2334
        loop->dump_head();
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2335
      }
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2336
#endif
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2337
    } else if (cl != NULL && loop->is_range_check_if(iff, this, invar)) {
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2338
      assert(proj->_con == predicate_proj->_con, "must match");
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2339
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2340
      // Range check for counted loops
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2341
      const Node*    cmp    = bol->in(1)->as_Cmp();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2342
      Node*          idx    = cmp->in(1);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2343
      assert(!invar.is_invariant(idx), "index is variant");
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2344
      assert(cmp->in(2)->Opcode() == Op_LoadRange || OptimizeFill, "must be");
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2345
      Node* rng = cmp->in(2);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2346
      assert(invar.is_invariant(rng), "range must be invariant");
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2347
      int scale    = 1;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2348
      Node* offset = zero;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2349
      bool ok = is_scaled_iv_plus_offset(idx, cl->phi(), &scale, &offset);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2350
      assert(ok, "must be index expression");
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2351
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2352
      Node* init    = cl->init_trip();
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2353
      Node* limit   = cl->limit();
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2354
      Node* stride  = cl->stride();
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2355
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2356
      // Build if's for the upper and lower bound tests.  The
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2357
      // lower_bound test will dominate the upper bound test and all
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2358
      // cloned or created nodes will use the lower bound test as
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2359
      // their declared control.
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2360
      ProjNode* lower_bound_proj = create_new_if_for_predicate(predicate_proj, NULL, Deoptimization::Reason_predicate);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2361
      ProjNode* upper_bound_proj = create_new_if_for_predicate(predicate_proj, NULL, Deoptimization::Reason_predicate);
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2362
      assert(upper_bound_proj->in(0)->as_If()->in(0) == lower_bound_proj, "should dominate");
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2363
      Node *ctrl = lower_bound_proj->in(0)->as_If()->in(0);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2364
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2365
      // Perform cloning to keep Invariance state correct since the
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2366
      // late schedule will place invariant things in the loop.
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2367
      rng = invar.clone(rng, ctrl);
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2368
      if (offset && offset != zero) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2369
        assert(invar.is_invariant(offset), "offset must be loop invariant");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2370
        offset = invar.clone(offset, ctrl);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2371
      }
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2372
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2373
      // Test the lower bound
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2374
      Node*  lower_bound_bol = rc_predicate(ctrl, scale, offset, init, limit, stride, rng, false);
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2375
      IfNode* lower_bound_iff = lower_bound_proj->in(0)->as_If();
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2376
      _igvn.hash_delete(lower_bound_iff);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2377
      lower_bound_iff->set_req(1, lower_bound_bol);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2378
      if (TraceLoopPredicate) tty->print_cr("lower bound check if: %d", lower_bound_iff->_idx);
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2379
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2380
      // Test the upper bound
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2381
      Node* upper_bound_bol = rc_predicate(ctrl, scale, offset, init, limit, stride, rng, true);
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2382
      IfNode* upper_bound_iff = upper_bound_proj->in(0)->as_If();
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2383
      _igvn.hash_delete(upper_bound_iff);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2384
      upper_bound_iff->set_req(1, upper_bound_bol);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2385
      if (TraceLoopPredicate) tty->print_cr("upper bound check if: %d", lower_bound_iff->_idx);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2386
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2387
      // Fall through into rest of the clean up code which will move
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2388
      // any dependent nodes onto the upper bound test.
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2389
      new_predicate_proj = upper_bound_proj;
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2390
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2391
#ifndef PRODUCT
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2392
      if (TraceLoopOpts && !TraceLoopPredicate) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2393
        tty->print("Predicate RC ");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2394
        loop->dump_head();
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2395
      }
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2396
#endif
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2397
    } else {
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2398
      // Loop variant check (for example, range check in non-counted loop)
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2399
      // with uncommon trap.
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2400
      continue;
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2401
    }
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2402
    assert(new_predicate_proj != NULL, "sanity");
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2403
    // Success - attach condition (new_predicate_bol) to predicate if
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2404
    invar.map_ctrl(proj, new_predicate_proj); // so that invariance test can be appropriate
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2405
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2406
    // Eliminate the old If in the loop body
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  2407
    dominated_by( new_predicate_proj, iff, proj->_con != new_predicate_proj->_con );
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2408
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2409
    hoisted = true;
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2410
    C->set_major_progress();
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2411
  } // end while
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2412
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2413
#ifndef PRODUCT
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2414
  // report that the loop predication has been actually performed
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2415
  // for this loop
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2416
  if (TraceLoopPredicate && hoisted) {
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2417
    tty->print("Loop Predication Performed:");
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2418
    loop->dump_head();
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2419
  }
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2420
#endif
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2421
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2422
  return hoisted;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2423
}
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2424
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2425
//------------------------------loop_predication--------------------------------
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2426
// driver routine for loop predication optimization
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2427
bool IdealLoopTree::loop_predication( PhaseIdealLoop *phase) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2428
  bool hoisted = false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2429
  // Recursively promote predicates
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2430
  if ( _child ) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2431
    hoisted = _child->loop_predication( phase);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2432
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2433
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2434
  // self
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2435
  if (!_irreducible && !tail()->is_top()) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2436
    hoisted |= phase->loop_predication_impl(this);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2437
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2438
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2439
  if ( _next ) { //sibling
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2440
    hoisted |= _next->loop_predication( phase);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2441
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2442
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2443
  return hoisted;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2444
}
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2445
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2446
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2447
// Process all the loops in the loop tree and replace any fill
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2448
// patterns with an intrisc version.
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2449
bool PhaseIdealLoop::do_intrinsify_fill() {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2450
  bool changed = false;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2451
  for (LoopTreeIterator iter(_ltree_root); !iter.done(); iter.next()) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2452
    IdealLoopTree* lpt = iter.current();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2453
    changed |= intrinsify_fill(lpt);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2454
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2455
  return changed;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2456
}
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2457
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2458
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2459
// Examine an inner loop looking for a a single store of an invariant
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2460
// value in a unit stride loop,
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2461
bool PhaseIdealLoop::match_fill_loop(IdealLoopTree* lpt, Node*& store, Node*& store_value,
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2462
                                     Node*& shift, Node*& con) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2463
  const char* msg = NULL;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2464
  Node* msg_node = NULL;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2465
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2466
  store_value = NULL;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2467
  con = NULL;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2468
  shift = NULL;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2469
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2470
  // Process the loop looking for stores.  If there are multiple
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2471
  // stores or extra control flow give at this point.
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2472
  CountedLoopNode* head = lpt->_head->as_CountedLoop();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2473
  for (uint i = 0; msg == NULL && i < lpt->_body.size(); i++) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2474
    Node* n = lpt->_body.at(i);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2475
    if (n->outcnt() == 0) continue; // Ignore dead
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2476
    if (n->is_Store()) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2477
      if (store != NULL) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2478
        msg = "multiple stores";
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2479
        break;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2480
      }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2481
      int opc = n->Opcode();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2482
      if (opc == Op_StoreP || opc == Op_StoreN || opc == Op_StoreCM) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2483
        msg = "oop fills not handled";
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2484
        break;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2485
      }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2486
      Node* value = n->in(MemNode::ValueIn);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2487
      if (!lpt->is_invariant(value)) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2488
        msg  = "variant store value";
6455
4a8142e5f75f 6982533: Crash in ~StubRoutines::jbyte_fill with AggressiveOpts enabled
never
parents: 6433
diff changeset
  2489
      } else if (!_igvn.type(n->in(MemNode::Address))->isa_aryptr()) {
4a8142e5f75f 6982533: Crash in ~StubRoutines::jbyte_fill with AggressiveOpts enabled
never
parents: 6433
diff changeset
  2490
        msg = "not array address";
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2491
      }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2492
      store = n;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2493
      store_value = value;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2494
    } else if (n->is_If() && n != head->loopexit()) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2495
      msg = "extra control flow";
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2496
      msg_node = n;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2497
    }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2498
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2499
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2500
  if (store == NULL) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2501
    // No store in loop
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2502
    return false;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2503
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2504
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2505
  if (msg == NULL && head->stride_con() != 1) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2506
    // could handle negative strides too
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2507
    if (head->stride_con() < 0) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2508
      msg = "negative stride";
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2509
    } else {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2510
      msg = "non-unit stride";
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2511
    }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2512
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2513
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2514
  if (msg == NULL && !store->in(MemNode::Address)->is_AddP()) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2515
    msg = "can't handle store address";
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2516
    msg_node = store->in(MemNode::Address);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2517
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2518
6739
750998abc20f 6984979: OptimizeFill misses some cases with an odd memory graph
never
parents: 6455
diff changeset
  2519
  if (msg == NULL &&
750998abc20f 6984979: OptimizeFill misses some cases with an odd memory graph
never
parents: 6455
diff changeset
  2520
      (!store->in(MemNode::Memory)->is_Phi() ||
750998abc20f 6984979: OptimizeFill misses some cases with an odd memory graph
never
parents: 6455
diff changeset
  2521
       store->in(MemNode::Memory)->in(LoopNode::LoopBackControl) != store)) {
750998abc20f 6984979: OptimizeFill misses some cases with an odd memory graph
never
parents: 6455
diff changeset
  2522
    msg = "store memory isn't proper phi";
750998abc20f 6984979: OptimizeFill misses some cases with an odd memory graph
never
parents: 6455
diff changeset
  2523
    msg_node = store->in(MemNode::Memory);
750998abc20f 6984979: OptimizeFill misses some cases with an odd memory graph
never
parents: 6455
diff changeset
  2524
  }
750998abc20f 6984979: OptimizeFill misses some cases with an odd memory graph
never
parents: 6455
diff changeset
  2525
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2526
  // Make sure there is an appropriate fill routine
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2527
  BasicType t = store->as_Mem()->memory_type();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2528
  const char* fill_name;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2529
  if (msg == NULL &&
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2530
      StubRoutines::select_fill_function(t, false, fill_name) == NULL) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2531
    msg = "unsupported store";
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2532
    msg_node = store;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2533
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2534
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2535
  if (msg != NULL) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2536
#ifndef PRODUCT
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2537
    if (TraceOptimizeFill) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2538
      tty->print_cr("not fill intrinsic candidate: %s", msg);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2539
      if (msg_node != NULL) msg_node->dump();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2540
    }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2541
#endif
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2542
    return false;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2543
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2544
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2545
  // Make sure the address expression can be handled.  It should be
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2546
  // head->phi * elsize + con.  head->phi might have a ConvI2L.
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2547
  Node* elements[4];
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2548
  Node* conv = NULL;
6455
4a8142e5f75f 6982533: Crash in ~StubRoutines::jbyte_fill with AggressiveOpts enabled
never
parents: 6433
diff changeset
  2549
  bool found_index = false;
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2550
  int count = store->in(MemNode::Address)->as_AddP()->unpack_offsets(elements, ARRAY_SIZE(elements));
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2551
  for (int e = 0; e < count; e++) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2552
    Node* n = elements[e];
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2553
    if (n->is_Con() && con == NULL) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2554
      con = n;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2555
    } else if (n->Opcode() == Op_LShiftX && shift == NULL) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2556
      Node* value = n->in(1);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2557
#ifdef _LP64
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2558
      if (value->Opcode() == Op_ConvI2L) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2559
        conv = value;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2560
        value = value->in(1);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2561
      }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2562
#endif
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2563
      if (value != head->phi()) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2564
        msg = "unhandled shift in address";
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2565
      } else {
6455
4a8142e5f75f 6982533: Crash in ~StubRoutines::jbyte_fill with AggressiveOpts enabled
never
parents: 6433
diff changeset
  2566
        found_index = true;
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2567
        shift = n;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2568
        assert(type2aelembytes(store->as_Mem()->memory_type(), true) == 1 << shift->in(2)->get_int(), "scale should match");
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2569
      }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2570
    } else if (n->Opcode() == Op_ConvI2L && conv == NULL) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2571
      if (n->in(1) == head->phi()) {
6455
4a8142e5f75f 6982533: Crash in ~StubRoutines::jbyte_fill with AggressiveOpts enabled
never
parents: 6433
diff changeset
  2572
        found_index = true;
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2573
        conv = n;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2574
      } else {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2575
        msg = "unhandled input to ConvI2L";
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2576
      }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2577
    } else if (n == head->phi()) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2578
      // no shift, check below for allowed cases
6455
4a8142e5f75f 6982533: Crash in ~StubRoutines::jbyte_fill with AggressiveOpts enabled
never
parents: 6433
diff changeset
  2579
      found_index = true;
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2580
    } else {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2581
      msg = "unhandled node in address";
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2582
      msg_node = n;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2583
    }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2584
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2585
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2586
  if (count == -1) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2587
    msg = "malformed address expression";
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2588
    msg_node = store;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2589
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2590
6455
4a8142e5f75f 6982533: Crash in ~StubRoutines::jbyte_fill with AggressiveOpts enabled
never
parents: 6433
diff changeset
  2591
  if (!found_index) {
4a8142e5f75f 6982533: Crash in ~StubRoutines::jbyte_fill with AggressiveOpts enabled
never
parents: 6433
diff changeset
  2592
    msg = "missing use of index";
4a8142e5f75f 6982533: Crash in ~StubRoutines::jbyte_fill with AggressiveOpts enabled
never
parents: 6433
diff changeset
  2593
  }
4a8142e5f75f 6982533: Crash in ~StubRoutines::jbyte_fill with AggressiveOpts enabled
never
parents: 6433
diff changeset
  2594
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2595
  // byte sized items won't have a shift
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2596
  if (msg == NULL && shift == NULL && t != T_BYTE && t != T_BOOLEAN) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2597
    msg = "can't find shift";
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2598
    msg_node = store;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2599
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2600
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2601
  if (msg != NULL) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2602
#ifndef PRODUCT
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2603
    if (TraceOptimizeFill) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2604
      tty->print_cr("not fill intrinsic: %s", msg);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2605
      if (msg_node != NULL) msg_node->dump();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2606
    }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2607
#endif
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2608
    return false;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2609
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2610
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2611
  // No make sure all the other nodes in the loop can be handled
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2612
  VectorSet ok(Thread::current()->resource_area());
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2613
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2614
  // store related values are ok
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2615
  ok.set(store->_idx);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2616
  ok.set(store->in(MemNode::Memory)->_idx);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2617
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2618
  // Loop structure is ok
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2619
  ok.set(head->_idx);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2620
  ok.set(head->loopexit()->_idx);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2621
  ok.set(head->phi()->_idx);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2622
  ok.set(head->incr()->_idx);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2623
  ok.set(head->loopexit()->cmp_node()->_idx);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2624
  ok.set(head->loopexit()->in(1)->_idx);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2625
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2626
  // Address elements are ok
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2627
  if (con)   ok.set(con->_idx);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2628
  if (shift) ok.set(shift->_idx);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2629
  if (conv)  ok.set(conv->_idx);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2630
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2631
  for (uint i = 0; msg == NULL && i < lpt->_body.size(); i++) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2632
    Node* n = lpt->_body.at(i);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2633
    if (n->outcnt() == 0) continue; // Ignore dead
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2634
    if (ok.test(n->_idx)) continue;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2635
    // Backedge projection is ok
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2636
    if (n->is_IfTrue() && n->in(0) == head->loopexit()) continue;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2637
    if (!n->is_AddP()) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2638
      msg = "unhandled node";
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2639
      msg_node = n;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2640
      break;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2641
    }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2642
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2643
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2644
  // Make sure no unexpected values are used outside the loop
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2645
  for (uint i = 0; msg == NULL && i < lpt->_body.size(); i++) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2646
    Node* n = lpt->_body.at(i);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2647
    // These values can be replaced with other nodes if they are used
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2648
    // outside the loop.
6739
750998abc20f 6984979: OptimizeFill misses some cases with an odd memory graph
never
parents: 6455
diff changeset
  2649
    if (n == store || n == head->loopexit() || n == head->incr() || n == store->in(MemNode::Memory)) continue;
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2650
    for (SimpleDUIterator iter(n); iter.has_next(); iter.next()) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2651
      Node* use = iter.get();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2652
      if (!lpt->_body.contains(use)) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2653
        msg = "node is used outside loop";
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2654
        // lpt->_body.dump();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2655
        msg_node = n;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2656
        break;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2657
      }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2658
    }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2659
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2660
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2661
#ifdef ASSERT
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2662
  if (TraceOptimizeFill) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2663
    if (msg != NULL) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2664
      tty->print_cr("no fill intrinsic: %s", msg);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2665
      if (msg_node != NULL) msg_node->dump();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2666
    } else {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2667
      tty->print_cr("fill intrinsic for:");
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2668
    }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2669
    store->dump();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2670
    if (Verbose) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2671
      lpt->_body.dump();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2672
    }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2673
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2674
#endif
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2675
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2676
  return msg == NULL;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2677
}
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2678
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2679
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2680
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2681
bool PhaseIdealLoop::intrinsify_fill(IdealLoopTree* lpt) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2682
  // Only for counted inner loops
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2683
  if (!lpt->is_counted() || !lpt->is_inner()) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2684
    return false;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2685
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2686
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2687
  // Must have constant stride
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2688
  CountedLoopNode* head = lpt->_head->as_CountedLoop();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2689
  if (!head->stride_is_con() || !head->is_normal_loop()) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2690
    return false;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2691
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2692
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2693
  // Check that the body only contains a store of a loop invariant
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2694
  // value that is indexed by the loop phi.
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2695
  Node* store = NULL;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2696
  Node* store_value = NULL;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2697
  Node* shift = NULL;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2698
  Node* offset = NULL;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2699
  if (!match_fill_loop(lpt, store, store_value, shift, offset)) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2700
    return false;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2701
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2702
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2703
  // Now replace the whole loop body by a call to a fill routine that
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2704
  // covers the same region as the loop.
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2705
  Node* base = store->in(MemNode::Address)->as_AddP()->in(AddPNode::Base);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2706
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2707
  // Build an expression for the beginning of the copy region
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2708
  Node* index = head->init_trip();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2709
#ifdef _LP64
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2710
  index = new (C, 2) ConvI2LNode(index);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2711
  _igvn.register_new_node_with_optimizer(index);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2712
#endif
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2713
  if (shift != NULL) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2714
    // byte arrays don't require a shift but others do.
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2715
    index = new (C, 3) LShiftXNode(index, shift->in(2));
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2716
    _igvn.register_new_node_with_optimizer(index);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2717
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2718
  index = new (C, 4) AddPNode(base, base, index);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2719
  _igvn.register_new_node_with_optimizer(index);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2720
  Node* from = new (C, 4) AddPNode(base, index, offset);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2721
  _igvn.register_new_node_with_optimizer(from);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2722
  // Compute the number of elements to copy
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2723
  Node* len = new (C, 3) SubINode(head->limit(), head->init_trip());
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2724
  _igvn.register_new_node_with_optimizer(len);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2725
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2726
  BasicType t = store->as_Mem()->memory_type();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2727
  bool aligned = false;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2728
  if (offset != NULL && head->init_trip()->is_Con()) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2729
    int element_size = type2aelembytes(t);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2730
    aligned = (offset->find_intptr_t_type()->get_con() + head->init_trip()->get_int() * element_size) % HeapWordSize == 0;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2731
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2732
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2733
  // Build a call to the fill routine
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2734
  const char* fill_name;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2735
  address fill = StubRoutines::select_fill_function(t, aligned, fill_name);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2736
  assert(fill != NULL, "what?");
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2737
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2738
  // Convert float/double to int/long for fill routines
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2739
  if (t == T_FLOAT) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2740
    store_value = new (C, 2) MoveF2INode(store_value);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2741
    _igvn.register_new_node_with_optimizer(store_value);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2742
  } else if (t == T_DOUBLE) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2743
    store_value = new (C, 2) MoveD2LNode(store_value);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2744
    _igvn.register_new_node_with_optimizer(store_value);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2745
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2746
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2747
  Node* mem_phi = store->in(MemNode::Memory);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2748
  Node* result_ctrl;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2749
  Node* result_mem;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2750
  const TypeFunc* call_type = OptoRuntime::array_fill_Type();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2751
  int size = call_type->domain()->cnt();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2752
  CallLeafNode *call = new (C, size) CallLeafNoFPNode(call_type, fill,
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2753
                                                      fill_name, TypeAryPtr::get_array_body_type(t));
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2754
  call->init_req(TypeFunc::Parms+0, from);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2755
  call->init_req(TypeFunc::Parms+1, store_value);
6770
a67870aaedb3 6980792: Crash "exception happened outside interpreter, nmethods and vtable stubs (1)"
never
parents: 6739
diff changeset
  2756
#ifdef _LP64
a67870aaedb3 6980792: Crash "exception happened outside interpreter, nmethods and vtable stubs (1)"
never
parents: 6739
diff changeset
  2757
  len = new (C, 2) ConvI2LNode(len);
a67870aaedb3 6980792: Crash "exception happened outside interpreter, nmethods and vtable stubs (1)"
never
parents: 6739
diff changeset
  2758
  _igvn.register_new_node_with_optimizer(len);
a67870aaedb3 6980792: Crash "exception happened outside interpreter, nmethods and vtable stubs (1)"
never
parents: 6739
diff changeset
  2759
#endif
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2760
  call->init_req(TypeFunc::Parms+2, len);
6770
a67870aaedb3 6980792: Crash "exception happened outside interpreter, nmethods and vtable stubs (1)"
never
parents: 6739
diff changeset
  2761
#ifdef _LP64
a67870aaedb3 6980792: Crash "exception happened outside interpreter, nmethods and vtable stubs (1)"
never
parents: 6739
diff changeset
  2762
  call->init_req(TypeFunc::Parms+3, C->top());
a67870aaedb3 6980792: Crash "exception happened outside interpreter, nmethods and vtable stubs (1)"
never
parents: 6739
diff changeset
  2763
#endif
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2764
  call->init_req( TypeFunc::Control, head->init_control());
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2765
  call->init_req( TypeFunc::I_O    , C->top() )        ;   // does no i/o
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2766
  call->init_req( TypeFunc::Memory ,  mem_phi->in(LoopNode::EntryControl) );
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2767
  call->init_req( TypeFunc::ReturnAdr, C->start()->proj_out(TypeFunc::ReturnAdr) );
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2768
  call->init_req( TypeFunc::FramePtr, C->start()->proj_out(TypeFunc::FramePtr) );
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2769
  _igvn.register_new_node_with_optimizer(call);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2770
  result_ctrl = new (C, 1) ProjNode(call,TypeFunc::Control);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2771
  _igvn.register_new_node_with_optimizer(result_ctrl);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2772
  result_mem = new (C, 1) ProjNode(call,TypeFunc::Memory);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2773
  _igvn.register_new_node_with_optimizer(result_mem);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2774
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2775
  // If this fill is tightly coupled to an allocation and overwrites
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2776
  // the whole body, allow it to take over the zeroing.
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2777
  AllocateNode* alloc = AllocateNode::Ideal_allocation(base, this);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2778
  if (alloc != NULL && alloc->is_AllocateArray()) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2779
    Node* length = alloc->as_AllocateArray()->Ideal_length();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2780
    if (head->limit() == length &&
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2781
        head->init_trip() == _igvn.intcon(0)) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2782
      if (TraceOptimizeFill) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2783
        tty->print_cr("Eliminated zeroing in allocation");
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2784
      }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2785
      alloc->maybe_set_complete(&_igvn);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2786
    } else {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2787
#ifdef ASSERT
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2788
      if (TraceOptimizeFill) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2789
        tty->print_cr("filling array but bounds don't match");
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2790
        alloc->dump();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2791
        head->init_trip()->dump();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2792
        head->limit()->dump();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2793
        length->dump();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2794
      }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2795
#endif
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2796
    }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2797
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2798
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2799
  // Redirect the old control and memory edges that are outside the loop.
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2800
  Node* exit = head->loopexit()->proj_out(0);
6739
750998abc20f 6984979: OptimizeFill misses some cases with an odd memory graph
never
parents: 6455
diff changeset
  2801
  // Sometimes the memory phi of the head is used as the outgoing
750998abc20f 6984979: OptimizeFill misses some cases with an odd memory graph
never
parents: 6455
diff changeset
  2802
  // state of the loop.  It's safe in this case to replace it with the
750998abc20f 6984979: OptimizeFill misses some cases with an odd memory graph
never
parents: 6455
diff changeset
  2803
  // result_mem.
750998abc20f 6984979: OptimizeFill misses some cases with an odd memory graph
never
parents: 6455
diff changeset
  2804
  _igvn.replace_node(store->in(MemNode::Memory), result_mem);
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2805
  _igvn.replace_node(exit, result_ctrl);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2806
  _igvn.replace_node(store, result_mem);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2807
  // Any uses the increment outside of the loop become the loop limit.
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2808
  _igvn.replace_node(head->incr(), head->limit());
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2809
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2810
  // Disconnect the head from the loop.
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2811
  for (uint i = 0; i < lpt->_body.size(); i++) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2812
    Node* n = lpt->_body.at(i);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2813
    _igvn.replace_node(n, C->top());
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2814
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2815
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2816
  return true;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2817
}