jdk-sandbox: comparison src/hotspot/share/opto/loopTransform.cpp

equal deleted inserted replaced

-:21b96ce2ed10
+:5209d8a6303e
 }
 //------------------------------compute_profile_trip_cnt----------------------------
 // Compute loop trip count from profile data as
 //    (backedge_count + loop_exit_count) / loop_exit_count
-void IdealLoopTree::compute_profile_trip_cnt( PhaseIdealLoop *phase ) {
-if (!_head->is_CountedLoop()) {
+float IdealLoopTree::compute_profile_trip_cnt_helper(Node* n) {
+if (n->is_If()) {
+IfNode *iff = n->as_If();
+if (iff->_fcnt != COUNT_UNKNOWN && iff->_prob != PROB_UNKNOWN) {
+Node *exit = is_loop_exit(iff);
+if (exit) {
+float exit_prob = iff->_prob;
+if (exit->Opcode() == Op_IfFalse) exit_prob = 1.0 - exit_prob;
+if (exit_prob > PROB_MIN) {
+float exit_cnt = iff->_fcnt * exit_prob;
+return exit_cnt;
+}
+}
+}
+}
+if (n->is_Jump()) {
+JumpNode *jmp = n->as_Jump();
+if (jmp->_fcnt != COUNT_UNKNOWN) {
+float* probs = jmp->_probs;
+float exit_prob = 0;
+PhaseIdealLoop *phase = _phase;
+for (DUIterator_Fast imax, i = jmp->fast_outs(imax); i < imax; i++) {
+JumpProjNode* u = jmp->fast_out(i)->as_JumpProj();
+if (!is_member(_phase->get_loop(u))) {
+exit_prob += probs[u->_con];
+}
+}
+return exit_prob * jmp->_fcnt;
+}
+}
+return 0;
+}
+void IdealLoopTree::compute_profile_trip_cnt(PhaseIdealLoop *phase) {
+if (!_head->is_Loop()) {
 return;
 }
-CountedLoopNode* head = _head->as_CountedLoop();
+LoopNode* head = _head->as_Loop();
 if (head->profile_trip_cnt() != COUNT_UNKNOWN) {
 return; // Already computed
 }
 float trip_cnt = (float)max_jint; // default is big
 while (back != head) {
 if ((back->Opcode() == Op_IfTrue || back->Opcode() == Op_IfFalse) &&
 back->in(0) &&
 back->in(0)->is_If() &&
 back->in(0)->as_If()->_fcnt != COUNT_UNKNOWN &&
-back->in(0)->as_If()->_prob != PROB_UNKNOWN) {
+back->in(0)->as_If()->_prob != PROB_UNKNOWN &&
+(back->Opcode() == Op_IfTrue ? 1-back->in(0)->as_If()->_prob : back->in(0)->as_If()->_prob) > PROB_MIN) {
 break;
 }
 back = phase->idom(back);
 }
 if (back != head) {
 assert((back->Opcode() == Op_IfTrue || back->Opcode() == Op_IfFalse) &&
 back->in(0), "if-projection exists");
 IfNode* back_if = back->in(0)->as_If();
-float loop_back_cnt = back_if->_fcnt * back_if->_prob;
+float loop_back_cnt = back_if->_fcnt * (back->Opcode() == Op_IfTrue ? back_if->_prob : (1 - back_if->_prob));
 // Now compute a loop exit count
 float loop_exit_cnt = 0.0f;
-for( uint i = 0; i < _body.size(); i++ ) {
+if (_child == NULL) {
-Node *n = _body[i];
+for( uint i = 0; i < _body.size(); i++ ) {
-if( n->is_If() ) {
+Node *n = _body[i];
-IfNode *iff = n->as_If();
+loop_exit_cnt += compute_profile_trip_cnt_helper(n);
-if( iff->_fcnt != COUNT_UNKNOWN && iff->_prob != PROB_UNKNOWN ) {
+}
-Node *exit = is_loop_exit(iff);
+} else {
-if( exit ) {
+ResourceMark rm;
-float exit_prob = iff->_prob;
+Unique_Node_List wq;
-if (exit->Opcode() == Op_IfFalse) exit_prob = 1.0 - exit_prob;
+wq.push(back);
-if (exit_prob > PROB_MIN) {
+for (uint i = 0; i < wq.size(); i++) {
-float exit_cnt = iff->_fcnt * exit_prob;
+Node *n = wq.at(i);
-loop_exit_cnt += exit_cnt;
+assert(n->is_CFG(), "only control nodes");
+if (n != head) {
+if (n->is_Region()) {
+for (uint j = 1; j < n->req(); j++) {
+wq.push(n->in(j));
 }
+} else {
+loop_exit_cnt += compute_profile_trip_cnt_helper(n);
+wq.push(n->in(0));
 }
 }
 }
 }
 if (loop_exit_cnt > 0.0f) {
 trip_cnt = (loop_back_cnt + loop_exit_cnt) / loop_exit_cnt;
 } else {
 // No exit count so use
 trip_cnt = loop_back_cnt;
 }
+} else {
+head->mark_profile_trip_failed();
 }
 #ifndef PRODUCT
 if (TraceProfileTripCount) {
 tty->print_cr("compute_profile_trip_cnt  lp: %d cnt: %f\n", head->_idx, trip_cnt);
 }
 // loop is never executed). When that happens, range check
 // CastII/ConvI2L nodes cause some data paths to die. For consistency,
 // the control paths must die too but the range checks were removed by
 // predication. The range checks that we add here guarantee that they
 // do.
-void PhaseIdealLoop::duplicate_predicates(CountedLoopNode* pre_head, Node* min_taken, Node* castii,
+void PhaseIdealLoop::duplicate_predicates_helper(Node* predicate, Node* castii, IdealLoopTree* outer_loop,
-IdealLoopTree* outer_loop, LoopNode* outer_main_head,
+LoopNode* outer_main_head, uint dd_main_head) {
-uint dd_main_head) {
+if (predicate != NULL) {
+IfNode* iff = predicate->in(0)->as_If();
+ProjNode* uncommon_proj = iff->proj_out(1 - predicate->as_Proj()->_con);
+Node* rgn = uncommon_proj->unique_ctrl_out();
+assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct");
+assert(iff->in(1)->in(1)->Opcode() == Op_Opaque1, "unexpected predicate shape");
+predicate = predicate->in(0)->in(0);
+Node* current_proj = outer_main_head->in(LoopNode::EntryControl);
+Node* prev_proj = current_proj;
+while (predicate != NULL && predicate->is_Proj() && predicate->in(0)->is_If()) {
+uncommon_proj = predicate->in(0)->as_If()->proj_out(1 - predicate->as_Proj()->_con);
+if (uncommon_proj->unique_ctrl_out() != rgn)
+break;
+iff = predicate->in(0)->as_If();
+if (iff->in(1)->Opcode() == Op_Opaque4) {
+Node_Stack to_clone(2);
+to_clone.push(iff->in(1), 1);
+uint current = C->unique();
+Node* result = NULL;
+// Look for the opaque node to replace with the init value
+// and clone everything in between. We keep the Opaque4 node
+// so the duplicated predicates are eliminated once loop
+// opts are over: they are here only to keep the IR graph
+// consistent.
+do {
+Node* n = to_clone.node();
+uint i = to_clone.index();
+Node* m = n->in(i);
+int op = m->Opcode();
+if (m->is_Bool() ||
+m->is_Cmp() ||
+op == Op_AndL ||
+op == Op_OrL ||
+op == Op_RShiftL ||
+op == Op_LShiftL ||
+op == Op_AddL ||
+op == Op_AddI ||
+op == Op_MulL ||
+op == Op_MulI ||
+op == Op_SubL ||
+op == Op_SubI ||
+op == Op_ConvI2L) {
+to_clone.push(m, 1);
+continue;
+}
+if (op == Op_Opaque1) {
+if (n->_idx < current) {
+n = n->clone();
+}
+n->set_req(i, castii);
+register_new_node(n, current_proj);
+to_clone.set_node(n);
+}
+for (;;) {
+Node* cur = to_clone.node();
+uint j = to_clone.index();
+if (j+1 < cur->req()) {
+to_clone.set_index(j+1);
+break;
+}
+to_clone.pop();
+if (to_clone.size() == 0) {
+result = cur;
+break;
+}
+Node* next = to_clone.node();
+j = to_clone.index();
+if (cur->_idx >= current) {
+if (next->_idx < current) {
+next = next->clone();
+register_new_node(next, current_proj);
+to_clone.set_node(next);
+}
+assert(next->in(j) != cur, "input should have been cloned");
+next->set_req(j, cur);
+}
+}
+} while (result == NULL);
+assert(result->_idx >= current, "new node expected");
+Node* proj = predicate->clone();
+Node* other_proj = uncommon_proj->clone();
+Node* new_iff = iff->clone();
+new_iff->set_req(1, result);
+proj->set_req(0, new_iff);
+other_proj->set_req(0, new_iff);
+Node *frame = new ParmNode(C->start(), TypeFunc::FramePtr);
+register_new_node(frame, C->start());
+// It's impossible for the predicate to fail at runtime. Use
+// an Halt node.
+Node* halt = new HaltNode(other_proj, frame);
+C->root()->add_req(halt);
+new_iff->set_req(0, prev_proj);
+register_control(new_iff, outer_loop->_parent, prev_proj);
+register_control(proj, outer_loop->_parent, new_iff);
+register_control(other_proj, _ltree_root, new_iff);
+register_control(halt, _ltree_root, other_proj);
+prev_proj = proj;
+}
+predicate = predicate->in(0)->in(0);
+}
+if (prev_proj != current_proj) {
+_igvn.replace_input_of(outer_main_head, LoopNode::EntryControl, prev_proj);
+set_idom(outer_main_head, prev_proj, dd_main_head);
+}
+}
+}
+void PhaseIdealLoop::duplicate_predicates(CountedLoopNode* pre_head, Node* castii, IdealLoopTree* outer_loop,
+LoopNode* outer_main_head, uint dd_main_head) {
 if (UseLoopPredicate) {
 Node* entry = pre_head->in(LoopNode::EntryControl);
 Node* predicate = NULL;
 predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
 if (predicate != NULL) {
 entry = entry->in(0)->in(0);
 }
+Node* profile_predicate = NULL;
+if (UseProfiledLoopPredicate) {
+profile_predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate);
+if (profile_predicate != NULL) {
+entry = skip_loop_predicates(entry);
+}
+}
 predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
-if (predicate != NULL) {
+duplicate_predicates_helper(predicate, castii, outer_loop, outer_main_head, dd_main_head);
-IfNode* iff = entry->in(0)->as_If();
+duplicate_predicates_helper(profile_predicate, castii, outer_loop, outer_main_head, dd_main_head);
-ProjNode* uncommon_proj = iff->proj_out(1 - entry->as_Proj()->_con);
-Node* rgn = uncommon_proj->unique_ctrl_out();
-assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct");
-assert(iff->in(1)->in(1)->Opcode() == Op_Opaque1, "unexpected predicate shape");
-entry = entry->in(0)->in(0);
-Node* prev_proj = min_taken;
-while (entry != NULL && entry->is_Proj() && entry->in(0)->is_If()) {
-uncommon_proj = entry->in(0)->as_If()->proj_out(1 - entry->as_Proj()->_con);
-if (uncommon_proj->unique_ctrl_out() != rgn)
-break;
-iff = entry->in(0)->as_If();
-if (iff->in(1)->Opcode() == Op_Opaque4) {
-Node_Stack to_clone(2);
-to_clone.push(iff->in(1), 1);
-uint current = C->unique();
-Node* result = NULL;
-// Look for the opaque node to replace with the init value
-// and clone everything in between. We keep the Opaque4 node
-// so the duplicated predicates are eliminated once loop
-// opts are over: they are here only to keep the IR graph
-// consistent.
-do {
-Node* n = to_clone.node();
-uint i = to_clone.index();
-Node* m = n->in(i);
-int op = m->Opcode();
-if (m->is_Bool() ||
-m->is_Cmp() ||
-op == Op_AndL ||
-op == Op_OrL ||
-op == Op_RShiftL ||
-op == Op_LShiftL ||
-op == Op_AddL ||
-op == Op_AddI ||
-op == Op_MulL ||
-op == Op_MulI ||
-op == Op_SubL ||
-op == Op_SubI ||
-op == Op_ConvI2L) {
-to_clone.push(m, 1);
-continue;
-}
-if (op == Op_Opaque1) {
-if (n->_idx < current) {
-n = n->clone();
-}
-n->set_req(i, castii);
-register_new_node(n, min_taken);
-to_clone.set_node(n);
-}
-for (;;) {
-Node* cur = to_clone.node();
-uint j = to_clone.index();
-if (j+1 < cur->req()) {
-to_clone.set_index(j+1);
-break;
-}
-to_clone.pop();
-if (to_clone.size() == 0) {
-result = cur;
-break;
-}
-Node* next = to_clone.node();
-j = to_clone.index();
-if (cur->_idx >= current) {
-if (next->_idx < current) {
-next = next->clone();
-register_new_node(next, min_taken);
-to_clone.set_node(next);
-}
-assert(next->in(j) != cur, "input should have been cloned");
-next->set_req(j, cur);
-}
-}
-} while (result == NULL);
-assert(result->_idx >= current, "new node expected");
-Node* proj = entry->clone();
-Node* other_proj = uncommon_proj->clone();
-Node* new_iff = iff->clone();
-new_iff->set_req(1, result);
-proj->set_req(0, new_iff);
-other_proj->set_req(0, new_iff);
-Node *frame = new ParmNode(C->start(), TypeFunc::FramePtr);
-register_new_node(frame, C->start());
-// It's impossible for the predicate to fail at runtime. Use
-// an Halt node.
-Node* halt = new HaltNode(other_proj, frame);
-C->root()->add_req(halt);
-new_iff->set_req(0, prev_proj);
-register_control(new_iff, outer_loop->_parent, prev_proj);
-register_control(proj, outer_loop->_parent, new_iff);
-register_control(other_proj, _ltree_root, new_iff);
-register_control(halt, _ltree_root, other_proj);
-prev_proj = proj;
-}
-entry = entry->in(0)->in(0);
-}
-_igvn.replace_input_of(outer_main_head, LoopNode::EntryControl, prev_proj);
-set_idom(outer_main_head, prev_proj, dd_main_head);
-}
 }
 }
 //------------------------------insert_pre_post_loops--------------------------
 // Insert pre and post loops.  If peel_only is set, the pre-loop can not have
 // dependencies.
 // CastII for the main loop:
 Node* castii = cast_incr_before_loop( pre_incr, min_taken, main_head );
 assert(castii != NULL, "no castII inserted");
-duplicate_predicates(pre_head, min_taken, castii, outer_loop, outer_main_head, dd_main_head);
+duplicate_predicates(pre_head, castii, outer_loop, outer_main_head, dd_main_head);
 // Step B4: Shorten the pre-loop to run only 1 iteration (for now).
 // RCE and alignment may change this later.
 Node *cmp_end = pre_end->cmp_node();
 assert( cmp_end->in(2) == limit, "" );
 needs_guard = (init_t->_lo <= limit_t->_hi);
 }
 }
 if (needs_guard) {
 // Check for an obvious zero trip guard.
-Node* inctrl = PhaseIdealLoop::skip_loop_predicates(cl->skip_predicates());
+Node* inctrl = PhaseIdealLoop::skip_all_loop_predicates(cl->skip_predicates());
 if (inctrl->Opcode() == Op_IfTrue || inctrl->Opcode() == Op_IfFalse) {
 bool maybe_swapped = (inctrl->Opcode() == Op_IfFalse);
 // The test should look like just the backedge of a CountedLoop
 Node* iff = inctrl->in(0);
 if (iff->is_If()) {

changeset 50623	5209d8a6303e
parent 50561	5756e8eecb17
child 50632	fd430e352427