hotspot/src/share/vm/opto/loopnode.cpp
changeset 13895 f6dfe4123709
parent 13524 bae2f51dfb73
child 13967 bf7c8dfb5121
equal deleted inserted replaced
13894:068a8efe7203 13895:f6dfe4123709
   341   // ---- SUCCESS!   Found A Trip-Counted Loop!  -----
   341   // ---- SUCCESS!   Found A Trip-Counted Loop!  -----
   342   //
   342   //
   343   assert(x->Opcode() == Op_Loop, "regular loops only");
   343   assert(x->Opcode() == Op_Loop, "regular loops only");
   344   C->print_method("Before CountedLoop", 3);
   344   C->print_method("Before CountedLoop", 3);
   345 
   345 
   346   Node *hook = new (C, 6) Node(6);
   346   Node *hook = new (C) Node(6);
   347 
   347 
   348   if (LoopLimitCheck) {
   348   if (LoopLimitCheck) {
   349 
   349 
   350   // ===================================================
   350   // ===================================================
   351   // Generate loop limit check to avoid integer overflow
   351   // Generate loop limit check to avoid integer overflow
   404     IfNode* check_iff = limit_check_proj->in(0)->as_If();
   404     IfNode* check_iff = limit_check_proj->in(0)->as_If();
   405     Node* cmp_limit;
   405     Node* cmp_limit;
   406     Node* bol;
   406     Node* bol;
   407 
   407 
   408     if (stride_con > 0) {
   408     if (stride_con > 0) {
   409       cmp_limit = new (C, 3) CmpINode(limit, _igvn.intcon(max_jint - stride_m));
   409       cmp_limit = new (C) CmpINode(limit, _igvn.intcon(max_jint - stride_m));
   410       bol = new (C, 2) BoolNode(cmp_limit, BoolTest::le);
   410       bol = new (C) BoolNode(cmp_limit, BoolTest::le);
   411     } else {
   411     } else {
   412       cmp_limit = new (C, 3) CmpINode(limit, _igvn.intcon(min_jint - stride_m));
   412       cmp_limit = new (C) CmpINode(limit, _igvn.intcon(min_jint - stride_m));
   413       bol = new (C, 2) BoolNode(cmp_limit, BoolTest::ge);
   413       bol = new (C) BoolNode(cmp_limit, BoolTest::ge);
   414     }
   414     }
   415     cmp_limit = _igvn.register_new_node_with_optimizer(cmp_limit);
   415     cmp_limit = _igvn.register_new_node_with_optimizer(cmp_limit);
   416     bol = _igvn.register_new_node_with_optimizer(bol);
   416     bol = _igvn.register_new_node_with_optimizer(bol);
   417     set_subtree_ctrl(bol);
   417     set_subtree_ctrl(bol);
   418 
   418 
   445     //
   445     //
   446     //   i = init; do {} while(i++ < limit);
   446     //   i = init; do {} while(i++ < limit);
   447     // is converted to
   447     // is converted to
   448     //   i = init; do {} while(++i < limit+1);
   448     //   i = init; do {} while(++i < limit+1);
   449     //
   449     //
   450     limit = gvn->transform(new (C, 3) AddINode(limit, stride));
   450     limit = gvn->transform(new (C) AddINode(limit, stride));
   451   }
   451   }
   452 
   452 
   453   // Now we need to canonicalize loop condition.
   453   // Now we need to canonicalize loop condition.
   454   if (bt == BoolTest::ne) {
   454   if (bt == BoolTest::ne) {
   455     assert(stride_con == 1 || stride_con == -1, "simple increment only");
   455     assert(stride_con == 1 || stride_con == -1, "simple increment only");
   464   if (incl_limit) {
   464   if (incl_limit) {
   465     // The limit check guaranties that 'limit <= (max_jint - stride)' so
   465     // The limit check guaranties that 'limit <= (max_jint - stride)' so
   466     // we can convert 'i <= limit' to 'i < limit+1' since stride != 0.
   466     // we can convert 'i <= limit' to 'i < limit+1' since stride != 0.
   467     //
   467     //
   468     Node* one = (stride_con > 0) ? gvn->intcon( 1) : gvn->intcon(-1);
   468     Node* one = (stride_con > 0) ? gvn->intcon( 1) : gvn->intcon(-1);
   469     limit = gvn->transform(new (C, 3) AddINode(limit, one));
   469     limit = gvn->transform(new (C) AddINode(limit, one));
   470     if (bt == BoolTest::le)
   470     if (bt == BoolTest::le)
   471       bt = BoolTest::lt;
   471       bt = BoolTest::lt;
   472     else if (bt == BoolTest::ge)
   472     else if (bt == BoolTest::ge)
   473       bt = BoolTest::gt;
   473       bt = BoolTest::gt;
   474     else
   474     else
   480 
   480 
   481   // If compare points to incr, we are ok.  Otherwise the compare
   481   // If compare points to incr, we are ok.  Otherwise the compare
   482   // can directly point to the phi; in this case adjust the compare so that
   482   // can directly point to the phi; in this case adjust the compare so that
   483   // it points to the incr by adjusting the limit.
   483   // it points to the incr by adjusting the limit.
   484   if (cmp->in(1) == phi || cmp->in(2) == phi)
   484   if (cmp->in(1) == phi || cmp->in(2) == phi)
   485     limit = gvn->transform(new (C, 3) AddINode(limit,stride));
   485     limit = gvn->transform(new (C) AddINode(limit,stride));
   486 
   486 
   487   // trip-count for +-tive stride should be: (limit - init_trip + stride - 1)/stride.
   487   // trip-count for +-tive stride should be: (limit - init_trip + stride - 1)/stride.
   488   // Final value for iterator should be: trip_count * stride + init_trip.
   488   // Final value for iterator should be: trip_count * stride + init_trip.
   489   Node *one_p = gvn->intcon( 1);
   489   Node *one_p = gvn->intcon( 1);
   490   Node *one_m = gvn->intcon(-1);
   490   Node *one_m = gvn->intcon(-1);
   493   switch( bt ) {
   493   switch( bt ) {
   494   case BoolTest::eq:
   494   case BoolTest::eq:
   495     ShouldNotReachHere();
   495     ShouldNotReachHere();
   496   case BoolTest::ne:            // Ahh, the case we desire
   496   case BoolTest::ne:            // Ahh, the case we desire
   497     if (stride_con == 1)
   497     if (stride_con == 1)
   498       trip_count = gvn->transform(new (C, 3) SubINode(limit,init_trip));
   498       trip_count = gvn->transform(new (C) SubINode(limit,init_trip));
   499     else if (stride_con == -1)
   499     else if (stride_con == -1)
   500       trip_count = gvn->transform(new (C, 3) SubINode(init_trip,limit));
   500       trip_count = gvn->transform(new (C) SubINode(init_trip,limit));
   501     else
   501     else
   502       ShouldNotReachHere();
   502       ShouldNotReachHere();
   503     set_subtree_ctrl(trip_count);
   503     set_subtree_ctrl(trip_count);
   504     //_loop.map(trip_count->_idx,loop(limit));
   504     //_loop.map(trip_count->_idx,loop(limit));
   505     break;
   505     break;
   506   case BoolTest::le:            // Maybe convert to '<' case
   506   case BoolTest::le:            // Maybe convert to '<' case
   507     limit = gvn->transform(new (C, 3) AddINode(limit,one_p));
   507     limit = gvn->transform(new (C) AddINode(limit,one_p));
   508     set_subtree_ctrl( limit );
   508     set_subtree_ctrl( limit );
   509     hook->init_req(4, limit);
   509     hook->init_req(4, limit);
   510 
   510 
   511     bt = BoolTest::lt;
   511     bt = BoolTest::lt;
   512     // Make the new limit be in the same loop nest as the old limit
   512     // Make the new limit be in the same loop nest as the old limit
   513     //_loop.map(limit->_idx,limit_loop);
   513     //_loop.map(limit->_idx,limit_loop);
   514     // Fall into next case
   514     // Fall into next case
   515   case BoolTest::lt: {          // Maybe convert to '!=' case
   515   case BoolTest::lt: {          // Maybe convert to '!=' case
   516     if (stride_con < 0) // Count down loop rolls through MAXINT
   516     if (stride_con < 0) // Count down loop rolls through MAXINT
   517       ShouldNotReachHere();
   517       ShouldNotReachHere();
   518     Node *range = gvn->transform(new (C, 3) SubINode(limit,init_trip));
   518     Node *range = gvn->transform(new (C) SubINode(limit,init_trip));
   519     set_subtree_ctrl( range );
   519     set_subtree_ctrl( range );
   520     hook->init_req(0, range);
   520     hook->init_req(0, range);
   521 
   521 
   522     Node *bias  = gvn->transform(new (C, 3) AddINode(range,stride));
   522     Node *bias  = gvn->transform(new (C) AddINode(range,stride));
   523     set_subtree_ctrl( bias );
   523     set_subtree_ctrl( bias );
   524     hook->init_req(1, bias);
   524     hook->init_req(1, bias);
   525 
   525 
   526     Node *bias1 = gvn->transform(new (C, 3) AddINode(bias,one_m));
   526     Node *bias1 = gvn->transform(new (C) AddINode(bias,one_m));
   527     set_subtree_ctrl( bias1 );
   527     set_subtree_ctrl( bias1 );
   528     hook->init_req(2, bias1);
   528     hook->init_req(2, bias1);
   529 
   529 
   530     trip_count  = gvn->transform(new (C, 3) DivINode(0,bias1,stride));
   530     trip_count  = gvn->transform(new (C) DivINode(0,bias1,stride));
   531     set_subtree_ctrl( trip_count );
   531     set_subtree_ctrl( trip_count );
   532     hook->init_req(3, trip_count);
   532     hook->init_req(3, trip_count);
   533     break;
   533     break;
   534   }
   534   }
   535 
   535 
   536   case BoolTest::ge:            // Maybe convert to '>' case
   536   case BoolTest::ge:            // Maybe convert to '>' case
   537     limit = gvn->transform(new (C, 3) AddINode(limit,one_m));
   537     limit = gvn->transform(new (C) AddINode(limit,one_m));
   538     set_subtree_ctrl( limit );
   538     set_subtree_ctrl( limit );
   539     hook->init_req(4 ,limit);
   539     hook->init_req(4 ,limit);
   540 
   540 
   541     bt = BoolTest::gt;
   541     bt = BoolTest::gt;
   542     // Make the new limit be in the same loop nest as the old limit
   542     // Make the new limit be in the same loop nest as the old limit
   543     //_loop.map(limit->_idx,limit_loop);
   543     //_loop.map(limit->_idx,limit_loop);
   544     // Fall into next case
   544     // Fall into next case
   545   case BoolTest::gt: {          // Maybe convert to '!=' case
   545   case BoolTest::gt: {          // Maybe convert to '!=' case
   546     if (stride_con > 0) // count up loop rolls through MININT
   546     if (stride_con > 0) // count up loop rolls through MININT
   547       ShouldNotReachHere();
   547       ShouldNotReachHere();
   548     Node *range = gvn->transform(new (C, 3) SubINode(limit,init_trip));
   548     Node *range = gvn->transform(new (C) SubINode(limit,init_trip));
   549     set_subtree_ctrl( range );
   549     set_subtree_ctrl( range );
   550     hook->init_req(0, range);
   550     hook->init_req(0, range);
   551 
   551 
   552     Node *bias  = gvn->transform(new (C, 3) AddINode(range,stride));
   552     Node *bias  = gvn->transform(new (C) AddINode(range,stride));
   553     set_subtree_ctrl( bias );
   553     set_subtree_ctrl( bias );
   554     hook->init_req(1, bias);
   554     hook->init_req(1, bias);
   555 
   555 
   556     Node *bias1 = gvn->transform(new (C, 3) AddINode(bias,one_p));
   556     Node *bias1 = gvn->transform(new (C) AddINode(bias,one_p));
   557     set_subtree_ctrl( bias1 );
   557     set_subtree_ctrl( bias1 );
   558     hook->init_req(2, bias1);
   558     hook->init_req(2, bias1);
   559 
   559 
   560     trip_count  = gvn->transform(new (C, 3) DivINode(0,bias1,stride));
   560     trip_count  = gvn->transform(new (C) DivINode(0,bias1,stride));
   561     set_subtree_ctrl( trip_count );
   561     set_subtree_ctrl( trip_count );
   562     hook->init_req(3, trip_count);
   562     hook->init_req(3, trip_count);
   563     break;
   563     break;
   564   }
   564   }
   565   } // switch( bt )
   565   } // switch( bt )
   566 
   566 
   567   Node *span = gvn->transform(new (C, 3) MulINode(trip_count,stride));
   567   Node *span = gvn->transform(new (C) MulINode(trip_count,stride));
   568   set_subtree_ctrl( span );
   568   set_subtree_ctrl( span );
   569   hook->init_req(5, span);
   569   hook->init_req(5, span);
   570 
   570 
   571   limit = gvn->transform(new (C, 3) AddINode(span,init_trip));
   571   limit = gvn->transform(new (C) AddINode(span,init_trip));
   572   set_subtree_ctrl( limit );
   572   set_subtree_ctrl( limit );
   573 
   573 
   574   } // LoopLimitCheck
   574   } // LoopLimitCheck
   575 
   575 
   576   // Check for SafePoint on backedge and remove
   576   // Check for SafePoint on backedge and remove
   615   test->set_req(1,cmp);
   615   test->set_req(1,cmp);
   616   _igvn.register_new_node_with_optimizer(test);
   616   _igvn.register_new_node_with_optimizer(test);
   617   set_ctrl(test, iff->in(0));
   617   set_ctrl(test, iff->in(0));
   618 
   618 
   619   // Replace the old IfNode with a new LoopEndNode
   619   // Replace the old IfNode with a new LoopEndNode
   620   Node *lex = _igvn.register_new_node_with_optimizer(new (C, 2) CountedLoopEndNode( iff->in(0), test, cl_prob, iff->as_If()->_fcnt ));
   620   Node *lex = _igvn.register_new_node_with_optimizer(new (C) CountedLoopEndNode( iff->in(0), test, cl_prob, iff->as_If()->_fcnt ));
   621   IfNode *le = lex->as_If();
   621   IfNode *le = lex->as_If();
   622   uint dd = dom_depth(iff);
   622   uint dd = dom_depth(iff);
   623   set_idom(le, le->in(0), dd); // Update dominance for loop exit
   623   set_idom(le, le->in(0), dd); // Update dominance for loop exit
   624   set_loop(le, loop);
   624   set_loop(le, loop);
   625 
   625 
   626   // Get the loop-exit control
   626   // Get the loop-exit control
   627   Node *iffalse = iff->as_If()->proj_out(!(iftrue_op == Op_IfTrue));
   627   Node *iffalse = iff->as_If()->proj_out(!(iftrue_op == Op_IfTrue));
   628 
   628 
   629   // Need to swap loop-exit and loop-back control?
   629   // Need to swap loop-exit and loop-back control?
   630   if (iftrue_op == Op_IfFalse) {
   630   if (iftrue_op == Op_IfFalse) {
   631     Node *ift2=_igvn.register_new_node_with_optimizer(new (C, 1) IfTrueNode (le));
   631     Node *ift2=_igvn.register_new_node_with_optimizer(new (C) IfTrueNode (le));
   632     Node *iff2=_igvn.register_new_node_with_optimizer(new (C, 1) IfFalseNode(le));
   632     Node *iff2=_igvn.register_new_node_with_optimizer(new (C) IfFalseNode(le));
   633 
   633 
   634     loop->_tail = back_control = ift2;
   634     loop->_tail = back_control = ift2;
   635     set_loop(ift2, loop);
   635     set_loop(ift2, loop);
   636     set_loop(iff2, get_loop(iffalse));
   636     set_loop(iff2, get_loop(iffalse));
   637 
   637 
   653   set_idom(iffalse, le, dd+1);
   653   set_idom(iffalse, le, dd+1);
   654   assert(iff->outcnt() == 0, "should be dead now");
   654   assert(iff->outcnt() == 0, "should be dead now");
   655   lazy_replace( iff, le ); // fix 'get_ctrl'
   655   lazy_replace( iff, le ); // fix 'get_ctrl'
   656 
   656 
   657   // Now setup a new CountedLoopNode to replace the existing LoopNode
   657   // Now setup a new CountedLoopNode to replace the existing LoopNode
   658   CountedLoopNode *l = new (C, 3) CountedLoopNode(init_control, back_control);
   658   CountedLoopNode *l = new (C) CountedLoopNode(init_control, back_control);
   659   l->set_unswitch_count(x->as_Loop()->unswitch_count()); // Preserve
   659   l->set_unswitch_count(x->as_Loop()->unswitch_count()); // Preserve
   660   // The following assert is approximately true, and defines the intention
   660   // The following assert is approximately true, and defines the intention
   661   // of can_be_counted_loop.  It fails, however, because phase->type
   661   // of can_be_counted_loop.  It fails, however, because phase->type
   662   // is not yet initialized for this loop and its parts.
   662   // is not yet initialized for this loop and its parts.
   663   //assert(l->can_be_counted_loop(this), "sanity");
   663   //assert(l->can_be_counted_loop(this), "sanity");
   727     // is counted and the limit was checked for overflow.
   727     // is counted and the limit was checked for overflow.
   728     assert(final_con == (long)final_int, "final value should be integer");
   728     assert(final_con == (long)final_int, "final value should be integer");
   729     limit = _igvn.intcon(final_int);
   729     limit = _igvn.intcon(final_int);
   730   } else {
   730   } else {
   731     // Create new LoopLimit node to get exact limit (final iv value).
   731     // Create new LoopLimit node to get exact limit (final iv value).
   732     limit = new (C, 4) LoopLimitNode(C, cl->init_trip(), cl->limit(), cl->stride());
   732     limit = new (C) LoopLimitNode(C, cl->init_trip(), cl->limit(), cl->stride());
   733     register_new_node(limit, cl->in(LoopNode::EntryControl));
   733     register_new_node(limit, cl->in(LoopNode::EntryControl));
   734   }
   734   }
   735   assert(limit != NULL, "sanity");
   735   assert(limit != NULL, "sanity");
   736   return limit;
   736   return limit;
   737 }
   737 }
   844   }
   844   }
   845   julong range = lim - ini + stride_p;
   845   julong range = lim - ini + stride_p;
   846   if (range <= max) {
   846   if (range <= max) {
   847     // Convert to integer expression if it is not overflow.
   847     // Convert to integer expression if it is not overflow.
   848     Node* stride_m = phase->intcon(stride_con - (stride_con > 0 ? 1 : -1));
   848     Node* stride_m = phase->intcon(stride_con - (stride_con > 0 ? 1 : -1));
   849     Node *range = phase->transform(new (phase->C, 3) SubINode(in(Limit), in(Init)));
   849     Node *range = phase->transform(new (phase->C) SubINode(in(Limit), in(Init)));
   850     Node *bias  = phase->transform(new (phase->C, 3) AddINode(range, stride_m));
   850     Node *bias  = phase->transform(new (phase->C) AddINode(range, stride_m));
   851     Node *trip  = phase->transform(new (phase->C, 3) DivINode(0, bias, in(Stride)));
   851     Node *trip  = phase->transform(new (phase->C) DivINode(0, bias, in(Stride)));
   852     Node *span  = phase->transform(new (phase->C, 3) MulINode(trip, in(Stride)));
   852     Node *span  = phase->transform(new (phase->C) MulINode(trip, in(Stride)));
   853     return new (phase->C, 3) AddINode(span, in(Init)); // exact limit
   853     return new (phase->C) AddINode(span, in(Init)); // exact limit
   854   }
   854   }
   855 
   855 
   856   if (is_power_of_2(stride_p) ||                // divisor is 2^n
   856   if (is_power_of_2(stride_p) ||                // divisor is 2^n
   857       !Matcher::has_match_rule(Op_LoopLimit)) { // or no specialized Mach node?
   857       !Matcher::has_match_rule(Op_LoopLimit)) { // or no specialized Mach node?
   858     // Convert to long expression to avoid integer overflow
   858     // Convert to long expression to avoid integer overflow
   859     // and let igvn optimizer convert this division.
   859     // and let igvn optimizer convert this division.
   860     //
   860     //
   861     Node*   init   = phase->transform( new (phase->C, 2) ConvI2LNode(in(Init)));
   861     Node*   init   = phase->transform( new (phase->C) ConvI2LNode(in(Init)));
   862     Node*  limit   = phase->transform( new (phase->C, 2) ConvI2LNode(in(Limit)));
   862     Node*  limit   = phase->transform( new (phase->C) ConvI2LNode(in(Limit)));
   863     Node* stride   = phase->longcon(stride_con);
   863     Node* stride   = phase->longcon(stride_con);
   864     Node* stride_m = phase->longcon(stride_con - (stride_con > 0 ? 1 : -1));
   864     Node* stride_m = phase->longcon(stride_con - (stride_con > 0 ? 1 : -1));
   865 
   865 
   866     Node *range = phase->transform(new (phase->C, 3) SubLNode(limit, init));
   866     Node *range = phase->transform(new (phase->C) SubLNode(limit, init));
   867     Node *bias  = phase->transform(new (phase->C, 3) AddLNode(range, stride_m));
   867     Node *bias  = phase->transform(new (phase->C) AddLNode(range, stride_m));
   868     Node *span;
   868     Node *span;
   869     if (stride_con > 0 && is_power_of_2(stride_p)) {
   869     if (stride_con > 0 && is_power_of_2(stride_p)) {
   870       // bias >= 0 if stride >0, so if stride is 2^n we can use &(-stride)
   870       // bias >= 0 if stride >0, so if stride is 2^n we can use &(-stride)
   871       // and avoid generating rounding for division. Zero trip guard should
   871       // and avoid generating rounding for division. Zero trip guard should
   872       // guarantee that init < limit but sometimes the guard is missing and
   872       // guarantee that init < limit but sometimes the guard is missing and
   873       // we can get situation when init > limit. Note, for the empty loop
   873       // we can get situation when init > limit. Note, for the empty loop
   874       // optimization zero trip guard is generated explicitly which leaves
   874       // optimization zero trip guard is generated explicitly which leaves
   875       // only RCE predicate where exact limit is used and the predicate
   875       // only RCE predicate where exact limit is used and the predicate
   876       // will simply fail forcing recompilation.
   876       // will simply fail forcing recompilation.
   877       Node* neg_stride   = phase->longcon(-stride_con);
   877       Node* neg_stride   = phase->longcon(-stride_con);
   878       span = phase->transform(new (phase->C, 3) AndLNode(bias, neg_stride));
   878       span = phase->transform(new (phase->C) AndLNode(bias, neg_stride));
   879     } else {
   879     } else {
   880       Node *trip  = phase->transform(new (phase->C, 3) DivLNode(0, bias, stride));
   880       Node *trip  = phase->transform(new (phase->C) DivLNode(0, bias, stride));
   881       span = phase->transform(new (phase->C, 3) MulLNode(trip, stride));
   881       span = phase->transform(new (phase->C) MulLNode(trip, stride));
   882     }
   882     }
   883     // Convert back to int
   883     // Convert back to int
   884     Node *span_int = phase->transform(new (phase->C, 2) ConvL2INode(span));
   884     Node *span_int = phase->transform(new (phase->C) ConvL2INode(span));
   885     return new (phase->C, 3) AddINode(span_int, in(Init)); // exact limit
   885     return new (phase->C) AddINode(span_int, in(Init)); // exact limit
   886   }
   886   }
   887 
   887 
   888   return NULL;    // No progress
   888   return NULL;    // No progress
   889 }
   889 }
   890 
   890 
  1086 void IdealLoopTree::split_fall_in( PhaseIdealLoop *phase, int fall_in_cnt ) {
  1086 void IdealLoopTree::split_fall_in( PhaseIdealLoop *phase, int fall_in_cnt ) {
  1087   PhaseIterGVN &igvn = phase->_igvn;
  1087   PhaseIterGVN &igvn = phase->_igvn;
  1088   uint i;
  1088   uint i;
  1089 
  1089 
  1090   // Make a new RegionNode to be the landing pad.
  1090   // Make a new RegionNode to be the landing pad.
  1091   Node *landing_pad = new (phase->C, fall_in_cnt+1) RegionNode( fall_in_cnt+1 );
  1091   Node *landing_pad = new (phase->C) RegionNode( fall_in_cnt+1 );
  1092   phase->set_loop(landing_pad,_parent);
  1092   phase->set_loop(landing_pad,_parent);
  1093   // Gather all the fall-in control paths into the landing pad
  1093   // Gather all the fall-in control paths into the landing pad
  1094   uint icnt = fall_in_cnt;
  1094   uint icnt = fall_in_cnt;
  1095   uint oreq = _head->req();
  1095   uint oreq = _head->req();
  1096   for( i = oreq-1; i>0; i-- )
  1096   for( i = oreq-1; i>0; i-- )
  1172   uint outer_idx = 1;
  1172   uint outer_idx = 1;
  1173   while( _head->in(outer_idx) != _tail ) outer_idx++;
  1173   while( _head->in(outer_idx) != _tail ) outer_idx++;
  1174 
  1174 
  1175   // Make a LoopNode for the outermost loop.
  1175   // Make a LoopNode for the outermost loop.
  1176   Node *ctl = _head->in(LoopNode::EntryControl);
  1176   Node *ctl = _head->in(LoopNode::EntryControl);
  1177   Node *outer = new (phase->C, 3) LoopNode( ctl, _head->in(outer_idx) );
  1177   Node *outer = new (phase->C) LoopNode( ctl, _head->in(outer_idx) );
  1178   outer = igvn.register_new_node_with_optimizer(outer, _head);
  1178   outer = igvn.register_new_node_with_optimizer(outer, _head);
  1179   phase->set_created_loop_node();
  1179   phase->set_created_loop_node();
  1180 
  1180 
  1181   // Outermost loop falls into '_head' loop
  1181   // Outermost loop falls into '_head' loop
  1182   _head->set_req(LoopNode::EntryControl, outer);
  1182   _head->set_req(LoopNode::EntryControl, outer);
  1286   // them all except optionally hot_idx.
  1286   // them all except optionally hot_idx.
  1287   PhaseIterGVN &igvn = phase->_igvn;
  1287   PhaseIterGVN &igvn = phase->_igvn;
  1288 
  1288 
  1289   Node *hot_tail = NULL;
  1289   Node *hot_tail = NULL;
  1290   // Make a Region for the merge point
  1290   // Make a Region for the merge point
  1291   Node *r = new (phase->C, 1) RegionNode(1);
  1291   Node *r = new (phase->C) RegionNode(1);
  1292   for( i = 2; i < _head->req(); i++ ) {
  1292   for( i = 2; i < _head->req(); i++ ) {
  1293     if( i != hot_idx )
  1293     if( i != hot_idx )
  1294       r->add_req( _head->in(i) );
  1294       r->add_req( _head->in(i) );
  1295     else hot_tail = _head->in(i);
  1295     else hot_tail = _head->in(i);
  1296   }
  1296   }
  1305     Node *out = _head->fast_out(j);
  1305     Node *out = _head->fast_out(j);
  1306     if( out->is_Phi() ) {
  1306     if( out->is_Phi() ) {
  1307       PhiNode* n = out->as_Phi();
  1307       PhiNode* n = out->as_Phi();
  1308       igvn.hash_delete(n);      // Delete from hash before hacking edges
  1308       igvn.hash_delete(n);      // Delete from hash before hacking edges
  1309       Node *hot_phi = NULL;
  1309       Node *hot_phi = NULL;
  1310       Node *phi = new (phase->C, r->req()) PhiNode(r, n->type(), n->adr_type());
  1310       Node *phi = new (phase->C) PhiNode(r, n->type(), n->adr_type());
  1311       // Check all inputs for the ones to peel out
  1311       // Check all inputs for the ones to peel out
  1312       uint j = 1;
  1312       uint j = 1;
  1313       for( uint i = 2; i < n->req(); i++ ) {
  1313       for( uint i = 2; i < n->req(); i++ ) {
  1314         if( i != hot_idx )
  1314         if( i != hot_idx )
  1315           phi->set_req( j++, n->in(i) );
  1315           phi->set_req( j++, n->in(i) );
  1427     split_outer_loop( phase );
  1427     split_outer_loop( phase );
  1428     result = true;
  1428     result = true;
  1429 
  1429 
  1430   } else if( !_head->is_Loop() && !_irreducible ) {
  1430   } else if( !_head->is_Loop() && !_irreducible ) {
  1431     // Make a new LoopNode to replace the old loop head
  1431     // Make a new LoopNode to replace the old loop head
  1432     Node *l = new (phase->C, 3) LoopNode( _head->in(1), _head->in(2) );
  1432     Node *l = new (phase->C) LoopNode( _head->in(1), _head->in(2) );
  1433     l = igvn.register_new_node_with_optimizer(l, _head);
  1433     l = igvn.register_new_node_with_optimizer(l, _head);
  1434     phase->set_created_loop_node();
  1434     phase->set_created_loop_node();
  1435     // Go ahead and replace _head
  1435     // Go ahead and replace _head
  1436     phase->_igvn.replace_node( _head, l );
  1436     phase->_igvn.replace_node( _head, l );
  1437     _head = l;
  1437     _head = l;
  1669       // variable differs from the trip counter by a loop-invariant
  1669       // variable differs from the trip counter by a loop-invariant
  1670       // amount, the difference between their respective initial values.
  1670       // amount, the difference between their respective initial values.
  1671       // It is scaled by the 'ratio_con'.
  1671       // It is scaled by the 'ratio_con'.
  1672       Node* ratio = _igvn.intcon(ratio_con);
  1672       Node* ratio = _igvn.intcon(ratio_con);
  1673       set_ctrl(ratio, C->root());
  1673       set_ctrl(ratio, C->root());
  1674       Node* ratio_init = new (C, 3) MulINode(init, ratio);
  1674       Node* ratio_init = new (C) MulINode(init, ratio);
  1675       _igvn.register_new_node_with_optimizer(ratio_init, init);
  1675       _igvn.register_new_node_with_optimizer(ratio_init, init);
  1676       set_early_ctrl(ratio_init);
  1676       set_early_ctrl(ratio_init);
  1677       Node* diff = new (C, 3) SubINode(init2, ratio_init);
  1677       Node* diff = new (C) SubINode(init2, ratio_init);
  1678       _igvn.register_new_node_with_optimizer(diff, init2);
  1678       _igvn.register_new_node_with_optimizer(diff, init2);
  1679       set_early_ctrl(diff);
  1679       set_early_ctrl(diff);
  1680       Node* ratio_idx = new (C, 3) MulINode(phi, ratio);
  1680       Node* ratio_idx = new (C) MulINode(phi, ratio);
  1681       _igvn.register_new_node_with_optimizer(ratio_idx, phi);
  1681       _igvn.register_new_node_with_optimizer(ratio_idx, phi);
  1682       set_ctrl(ratio_idx, cl);
  1682       set_ctrl(ratio_idx, cl);
  1683       Node* add = new (C, 3) AddINode(ratio_idx, diff);
  1683       Node* add = new (C) AddINode(ratio_idx, diff);
  1684       _igvn.register_new_node_with_optimizer(add);
  1684       _igvn.register_new_node_with_optimizer(add);
  1685       set_ctrl(add, cl);
  1685       set_ctrl(add, cl);
  1686       _igvn.replace_node( phi2, add );
  1686       _igvn.replace_node( phi2, add );
  1687       // Sometimes an induction variable is unused
  1687       // Sometimes an induction variable is unused
  1688       if (add->outcnt() == 0) {
  1688       if (add->outcnt() == 0) {
  2675         // optimizing an infinite loop?
  2675         // optimizing an infinite loop?
  2676         l = _ltree_root;        // Oops, found infinite loop
  2676         l = _ltree_root;        // Oops, found infinite loop
  2677 
  2677 
  2678         if (!_verify_only) {
  2678         if (!_verify_only) {
  2679           // Insert the NeverBranch between 'm' and it's control user.
  2679           // Insert the NeverBranch between 'm' and it's control user.
  2680           NeverBranchNode *iff = new (C, 1) NeverBranchNode( m );
  2680           NeverBranchNode *iff = new (C) NeverBranchNode( m );
  2681           _igvn.register_new_node_with_optimizer(iff);
  2681           _igvn.register_new_node_with_optimizer(iff);
  2682           set_loop(iff, l);
  2682           set_loop(iff, l);
  2683           Node *if_t = new (C, 1) CProjNode( iff, 0 );
  2683           Node *if_t = new (C) CProjNode( iff, 0 );
  2684           _igvn.register_new_node_with_optimizer(if_t);
  2684           _igvn.register_new_node_with_optimizer(if_t);
  2685           set_loop(if_t, l);
  2685           set_loop(if_t, l);
  2686 
  2686 
  2687           Node* cfg = NULL;       // Find the One True Control User of m
  2687           Node* cfg = NULL;       // Find the One True Control User of m
  2688           for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax; j++) {
  2688           for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax; j++) {
  2694           uint k = 0;             // Probably cfg->in(0)
  2694           uint k = 0;             // Probably cfg->in(0)
  2695           while( cfg->in(k) != m ) k++; // But check incase cfg is a Region
  2695           while( cfg->in(k) != m ) k++; // But check incase cfg is a Region
  2696           cfg->set_req( k, if_t ); // Now point to NeverBranch
  2696           cfg->set_req( k, if_t ); // Now point to NeverBranch
  2697 
  2697 
  2698           // Now create the never-taken loop exit
  2698           // Now create the never-taken loop exit
  2699           Node *if_f = new (C, 1) CProjNode( iff, 1 );
  2699           Node *if_f = new (C) CProjNode( iff, 1 );
  2700           _igvn.register_new_node_with_optimizer(if_f);
  2700           _igvn.register_new_node_with_optimizer(if_f);
  2701           set_loop(if_f, l);
  2701           set_loop(if_f, l);
  2702           // Find frame ptr for Halt.  Relies on the optimizer
  2702           // Find frame ptr for Halt.  Relies on the optimizer
  2703           // V-N'ing.  Easier and quicker than searching through
  2703           // V-N'ing.  Easier and quicker than searching through
  2704           // the program structure.
  2704           // the program structure.
  2705           Node *frame = new (C, 1) ParmNode( C->start(), TypeFunc::FramePtr );
  2705           Node *frame = new (C) ParmNode( C->start(), TypeFunc::FramePtr );
  2706           _igvn.register_new_node_with_optimizer(frame);
  2706           _igvn.register_new_node_with_optimizer(frame);
  2707           // Halt & Catch Fire
  2707           // Halt & Catch Fire
  2708           Node *halt = new (C, TypeFunc::Parms) HaltNode( if_f, frame );
  2708           Node *halt = new (C) HaltNode( if_f, frame );
  2709           _igvn.register_new_node_with_optimizer(halt);
  2709           _igvn.register_new_node_with_optimizer(halt);
  2710           set_loop(halt, l);
  2710           set_loop(halt, l);
  2711           C->root()->add_req(halt);
  2711           C->root()->add_req(halt);
  2712         }
  2712         }
  2713         set_loop(C->root(), _ltree_root);
  2713         set_loop(C->root(), _ltree_root);