54 Node *phi; |
54 Node *phi; |
55 if (t_oop != NULL && t_oop->is_known_instance_field()) { |
55 if (t_oop != NULL && t_oop->is_known_instance_field()) { |
56 int iid = t_oop->instance_id(); |
56 int iid = t_oop->instance_id(); |
57 int index = C->get_alias_index(t_oop); |
57 int index = C->get_alias_index(t_oop); |
58 int offset = t_oop->offset(); |
58 int offset = t_oop->offset(); |
59 phi = new (C) PhiNode(region, type, NULL, iid, index, offset); |
59 phi = new PhiNode(region, type, NULL, iid, index, offset); |
60 } else { |
60 } else { |
61 phi = PhiNode::make_blank(region, n); |
61 phi = PhiNode::make_blank(region, n); |
62 } |
62 } |
63 uint old_unique = C->unique(); |
63 uint old_unique = C->unique(); |
64 for (uint i = 1; i < region->req(); i++) { |
64 for (uint i = 1; i < region->req(); i++) { |
361 // Convert I-V into I+ (0-V); same for V-I |
361 // Convert I-V into I+ (0-V); same for V-I |
362 if( add->Opcode() == Op_SubI && |
362 if( add->Opcode() == Op_SubI && |
363 _igvn.type( add->in(1) ) != TypeInt::ZERO ) { |
363 _igvn.type( add->in(1) ) != TypeInt::ZERO ) { |
364 Node *zero = _igvn.intcon(0); |
364 Node *zero = _igvn.intcon(0); |
365 set_ctrl(zero, C->root()); |
365 set_ctrl(zero, C->root()); |
366 Node *neg = new (C) SubINode( _igvn.intcon(0), add->in(2) ); |
366 Node *neg = new SubINode( _igvn.intcon(0), add->in(2) ); |
367 register_new_node( neg, get_ctrl(add->in(2) ) ); |
367 register_new_node( neg, get_ctrl(add->in(2) ) ); |
368 add = new (C) AddINode( add->in(1), neg ); |
368 add = new AddINode( add->in(1), neg ); |
369 register_new_node( add, add_ctrl ); |
369 register_new_node( add, add_ctrl ); |
370 } |
370 } |
371 if( add->Opcode() != Op_AddI ) return NULL; |
371 if( add->Opcode() != Op_AddI ) return NULL; |
372 // See if one add input is loop invariant |
372 // See if one add input is loop invariant |
373 Node *add_var = add->in(1); |
373 Node *add_var = add->in(1); |
389 return NULL; |
389 return NULL; |
390 if( n_loop == add_invar_loop || !add_invar_loop->is_member( n_loop ) ) |
390 if( n_loop == add_invar_loop || !add_invar_loop->is_member( n_loop ) ) |
391 return NULL; // No invariant part of the add? |
391 return NULL; // No invariant part of the add? |
392 |
392 |
393 // Yes! Reshape address expression! |
393 // Yes! Reshape address expression! |
394 Node *inv_scale = new (C) LShiftINode( add_invar, scale ); |
394 Node *inv_scale = new LShiftINode( add_invar, scale ); |
395 Node *inv_scale_ctrl = |
395 Node *inv_scale_ctrl = |
396 dom_depth(add_invar_ctrl) > dom_depth(scale_ctrl) ? |
396 dom_depth(add_invar_ctrl) > dom_depth(scale_ctrl) ? |
397 add_invar_ctrl : scale_ctrl; |
397 add_invar_ctrl : scale_ctrl; |
398 register_new_node( inv_scale, inv_scale_ctrl ); |
398 register_new_node( inv_scale, inv_scale_ctrl ); |
399 Node *var_scale = new (C) LShiftINode( add_var, scale ); |
399 Node *var_scale = new LShiftINode( add_var, scale ); |
400 register_new_node( var_scale, n_ctrl ); |
400 register_new_node( var_scale, n_ctrl ); |
401 Node *var_add = new (C) AddINode( var_scale, inv_scale ); |
401 Node *var_add = new AddINode( var_scale, inv_scale ); |
402 register_new_node( var_add, n_ctrl ); |
402 register_new_node( var_add, n_ctrl ); |
403 _igvn.replace_node( n, var_add ); |
403 _igvn.replace_node( n, var_add ); |
404 return var_add; |
404 return var_add; |
405 } |
405 } |
406 |
406 |
428 Node *n23_ctrl = get_ctrl(n->in(2)->in(3)); |
428 Node *n23_ctrl = get_ctrl(n->in(2)->in(3)); |
429 IdealLoopTree *n22loop = get_loop( n22_ctrl ); |
429 IdealLoopTree *n22loop = get_loop( n22_ctrl ); |
430 IdealLoopTree *n23_loop = get_loop( n23_ctrl ); |
430 IdealLoopTree *n23_loop = get_loop( n23_ctrl ); |
431 if( n22loop != n_loop && n22loop->is_member(n_loop) && |
431 if( n22loop != n_loop && n22loop->is_member(n_loop) && |
432 n23_loop == n_loop ) { |
432 n23_loop == n_loop ) { |
433 Node *add1 = new (C) AddPNode( n->in(1), n->in(2)->in(2), n->in(3) ); |
433 Node *add1 = new AddPNode( n->in(1), n->in(2)->in(2), n->in(3) ); |
434 // Stuff new AddP in the loop preheader |
434 // Stuff new AddP in the loop preheader |
435 register_new_node( add1, n_loop->_head->in(LoopNode::EntryControl) ); |
435 register_new_node( add1, n_loop->_head->in(LoopNode::EntryControl) ); |
436 Node *add2 = new (C) AddPNode( n->in(1), add1, n->in(2)->in(3) ); |
436 Node *add2 = new AddPNode( n->in(1), add1, n->in(2)->in(3) ); |
437 register_new_node( add2, n_ctrl ); |
437 register_new_node( add2, n_ctrl ); |
438 _igvn.replace_node( n, add2 ); |
438 _igvn.replace_node( n, add2 ); |
439 return add2; |
439 return add2; |
440 } |
440 } |
441 } |
441 } |
449 if( is_member(n_loop,get_ctrl(V)) ) { |
449 if( is_member(n_loop,get_ctrl(V)) ) { |
450 } else { |
450 } else { |
451 Node *tmp = V; V = I; I = tmp; |
451 Node *tmp = V; V = I; I = tmp; |
452 } |
452 } |
453 if( !is_member(n_loop,get_ctrl(I)) ) { |
453 if( !is_member(n_loop,get_ctrl(I)) ) { |
454 Node *add1 = new (C) AddPNode( n->in(1), n->in(2), I ); |
454 Node *add1 = new AddPNode( n->in(1), n->in(2), I ); |
455 // Stuff new AddP in the loop preheader |
455 // Stuff new AddP in the loop preheader |
456 register_new_node( add1, n_loop->_head->in(LoopNode::EntryControl) ); |
456 register_new_node( add1, n_loop->_head->in(LoopNode::EntryControl) ); |
457 Node *add2 = new (C) AddPNode( n->in(1), add1, V ); |
457 Node *add2 = new AddPNode( n->in(1), add1, V ); |
458 register_new_node( add2, n_ctrl ); |
458 register_new_node( add2, n_ctrl ); |
459 _igvn.replace_node( n, add2 ); |
459 _igvn.replace_node( n, add2 ); |
460 return add2; |
460 return add2; |
461 } |
461 } |
462 } |
462 } |
1102 |
1102 |
1103 Node *sample_bool = phi->in(1); |
1103 Node *sample_bool = phi->in(1); |
1104 Node *sample_cmp = sample_bool->in(1); |
1104 Node *sample_cmp = sample_bool->in(1); |
1105 |
1105 |
1106 // Make Phis to merge the Cmp's inputs. |
1106 // Make Phis to merge the Cmp's inputs. |
1107 PhiNode *phi1 = new (C) PhiNode( phi->in(0), Type::TOP ); |
1107 PhiNode *phi1 = new PhiNode( phi->in(0), Type::TOP ); |
1108 PhiNode *phi2 = new (C) PhiNode( phi->in(0), Type::TOP ); |
1108 PhiNode *phi2 = new PhiNode( phi->in(0), Type::TOP ); |
1109 for( i = 1; i < phi->req(); i++ ) { |
1109 for( i = 1; i < phi->req(); i++ ) { |
1110 Node *n1 = phi->in(i)->in(1)->in(1); |
1110 Node *n1 = phi->in(i)->in(1)->in(1); |
1111 Node *n2 = phi->in(i)->in(1)->in(2); |
1111 Node *n2 = phi->in(i)->in(1)->in(2); |
1112 phi1->set_req( i, n1 ); |
1112 phi1->set_req( i, n1 ); |
1113 phi2->set_req( i, n2 ); |
1113 phi2->set_req( i, n2 ); |
1170 } |
1170 } |
1171 |
1171 |
1172 Node *sample_cmp = phi->in(1); |
1172 Node *sample_cmp = phi->in(1); |
1173 |
1173 |
1174 // Make Phis to merge the Cmp's inputs. |
1174 // Make Phis to merge the Cmp's inputs. |
1175 PhiNode *phi1 = new (C) PhiNode( phi->in(0), Type::TOP ); |
1175 PhiNode *phi1 = new PhiNode( phi->in(0), Type::TOP ); |
1176 PhiNode *phi2 = new (C) PhiNode( phi->in(0), Type::TOP ); |
1176 PhiNode *phi2 = new PhiNode( phi->in(0), Type::TOP ); |
1177 for( uint j = 1; j < phi->req(); j++ ) { |
1177 for( uint j = 1; j < phi->req(); j++ ) { |
1178 Node *cmp_top = phi->in(j); // Inputs are all Cmp or TOP |
1178 Node *cmp_top = phi->in(j); // Inputs are all Cmp or TOP |
1179 Node *n1, *n2; |
1179 Node *n1, *n2; |
1180 if( cmp_top->is_Cmp() ) { |
1180 if( cmp_top->is_Cmp() ) { |
1181 n1 = cmp_top->in(1); |
1181 n1 = cmp_top->in(1); |
1335 set_loop(newuse, use_loop); |
1335 set_loop(newuse, use_loop); |
1336 set_idom(newuse, nnn, dom_depth(nnn) + 1 ); |
1336 set_idom(newuse, nnn, dom_depth(nnn) + 1 ); |
1337 |
1337 |
1338 // We need a Region to merge the exit from the peeled body and the |
1338 // We need a Region to merge the exit from the peeled body and the |
1339 // exit from the old loop body. |
1339 // exit from the old loop body. |
1340 RegionNode *r = new (C) RegionNode(3); |
1340 RegionNode *r = new RegionNode(3); |
1341 // Map the old use to the new merge point |
1341 // Map the old use to the new merge point |
1342 old_new.map( use->_idx, r ); |
1342 old_new.map( use->_idx, r ); |
1343 uint dd_r = MIN2(dom_depth(newuse),dom_depth(use)); |
1343 uint dd_r = MIN2(dom_depth(newuse),dom_depth(use)); |
1344 assert( dd_r >= dom_depth(dom_lca(newuse,use)), "" ); |
1344 assert( dd_r >= dom_depth(dom_lca(newuse,use)), "" ); |
1345 |
1345 |
1682 |
1682 |
1683 proj->set_req(0, NULL); // temporary disconnect |
1683 proj->set_req(0, NULL); // temporary disconnect |
1684 ProjNode* proj2 = proj_clone(proj, iff); |
1684 ProjNode* proj2 = proj_clone(proj, iff); |
1685 register_node(proj2, loop, iff, ddepth); |
1685 register_node(proj2, loop, iff, ddepth); |
1686 |
1686 |
1687 Node* cmp = Signed ? (Node*) new (C)CmpINode(left, right) : (Node*) new (C)CmpUNode(left, right); |
1687 Node* cmp = Signed ? (Node*) new CmpINode(left, right) : (Node*) new CmpUNode(left, right); |
1688 register_node(cmp, loop, proj2, ddepth); |
1688 register_node(cmp, loop, proj2, ddepth); |
1689 |
1689 |
1690 BoolNode* bol = new (C)BoolNode(cmp, relop); |
1690 BoolNode* bol = new BoolNode(cmp, relop); |
1691 register_node(bol, loop, proj2, ddepth); |
1691 register_node(bol, loop, proj2, ddepth); |
1692 |
1692 |
1693 IfNode* new_if = new (C)IfNode(proj2, bol, iff->_prob, iff->_fcnt); |
1693 IfNode* new_if = new IfNode(proj2, bol, iff->_prob, iff->_fcnt); |
1694 register_node(new_if, loop, proj2, ddepth); |
1694 register_node(new_if, loop, proj2, ddepth); |
1695 |
1695 |
1696 proj->set_req(0, new_if); // reattach |
1696 proj->set_req(0, new_if); // reattach |
1697 set_idom(proj, new_if, ddepth); |
1697 set_idom(proj, new_if, ddepth); |
1698 |
1698 |
1740 |
1740 |
1741 proj->set_req(0, NULL); // temporary disconnect |
1741 proj->set_req(0, NULL); // temporary disconnect |
1742 ProjNode* proj2 = proj_clone(proj, iff); |
1742 ProjNode* proj2 = proj_clone(proj, iff); |
1743 register_node(proj2, loop, iff, ddepth); |
1743 register_node(proj2, loop, iff, ddepth); |
1744 |
1744 |
1745 RegionNode* reg = new (C)RegionNode(2); |
1745 RegionNode* reg = new RegionNode(2); |
1746 reg->set_req(1, proj2); |
1746 reg->set_req(1, proj2); |
1747 register_node(reg, loop, iff, ddepth); |
1747 register_node(reg, loop, iff, ddepth); |
1748 |
1748 |
1749 IfNode* dum_if = new (C)IfNode(reg, short_circuit_if(NULL, proj), iff->_prob, iff->_fcnt); |
1749 IfNode* dum_if = new IfNode(reg, short_circuit_if(NULL, proj), iff->_prob, iff->_fcnt); |
1750 register_node(dum_if, loop, reg, ddepth); |
1750 register_node(dum_if, loop, reg, ddepth); |
1751 |
1751 |
1752 proj->set_req(0, dum_if); // reattach |
1752 proj->set_req(0, dum_if); // reattach |
1753 set_idom(proj, dum_if, ddepth); |
1753 set_idom(proj, dum_if, ddepth); |
1754 |
1754 |
2567 |
2567 |
2568 // Step 3: clone loop, retarget control, and insert new phis |
2568 // Step 3: clone loop, retarget control, and insert new phis |
2569 |
2569 |
2570 // Create new loop head for new phis and to hang |
2570 // Create new loop head for new phis and to hang |
2571 // the nodes being moved (sinked) from the peel region. |
2571 // the nodes being moved (sinked) from the peel region. |
2572 LoopNode* new_head = new (C) LoopNode(last_peel, last_peel); |
2572 LoopNode* new_head = new LoopNode(last_peel, last_peel); |
2573 new_head->set_unswitch_count(head->unswitch_count()); // Preserve |
2573 new_head->set_unswitch_count(head->unswitch_count()); // Preserve |
2574 _igvn.register_new_node_with_optimizer(new_head); |
2574 _igvn.register_new_node_with_optimizer(new_head); |
2575 assert(first_not_peeled->in(0) == last_peel, "last_peel <- first_not_peeled"); |
2575 assert(first_not_peeled->in(0) == last_peel, "last_peel <- first_not_peeled"); |
2576 first_not_peeled->set_req(0, new_head); |
2576 first_not_peeled->set_req(0, new_head); |
2577 set_loop(new_head, loop); |
2577 set_loop(new_head, loop); |
2767 // Check that use is live out the bottom. Assuming the trip-counter |
2767 // Check that use is live out the bottom. Assuming the trip-counter |
2768 // update is right at the bottom, uses of of the loop middle are ok. |
2768 // update is right at the bottom, uses of of the loop middle are ok. |
2769 if (dom_lca(exit, u_ctrl) != exit) continue; |
2769 if (dom_lca(exit, u_ctrl) != exit) continue; |
2770 // Hit! Refactor use to use the post-incremented tripcounter. |
2770 // Hit! Refactor use to use the post-incremented tripcounter. |
2771 // Compute a post-increment tripcounter. |
2771 // Compute a post-increment tripcounter. |
2772 Node *opaq = new (C) Opaque2Node( C, cle->incr() ); |
2772 Node *opaq = new Opaque2Node( C, cle->incr() ); |
2773 register_new_node( opaq, u_ctrl ); |
2773 register_new_node( opaq, u_ctrl ); |
2774 Node *neg_stride = _igvn.intcon(-cle->stride_con()); |
2774 Node *neg_stride = _igvn.intcon(-cle->stride_con()); |
2775 set_ctrl(neg_stride, C->root()); |
2775 set_ctrl(neg_stride, C->root()); |
2776 Node *post = new (C) AddINode( opaq, neg_stride); |
2776 Node *post = new AddINode( opaq, neg_stride); |
2777 register_new_node( post, u_ctrl ); |
2777 register_new_node( post, u_ctrl ); |
2778 _igvn.rehash_node_delayed(use); |
2778 _igvn.rehash_node_delayed(use); |
2779 for (uint j = 1; j < use->req(); j++) { |
2779 for (uint j = 1; j < use->req(); j++) { |
2780 if (use->in(j) == phi) |
2780 if (use->in(j) == phi) |
2781 use->set_req(j, post); |
2781 use->set_req(j, post); |