299 // Interface from PhaseIdealLoop |
299 // Interface from PhaseIdealLoop |
300 Node* PhaseIdealLoop::clone_loop_predicates(Node* old_entry, Node* new_entry, bool clone_limit_check) { |
300 Node* PhaseIdealLoop::clone_loop_predicates(Node* old_entry, Node* new_entry, bool clone_limit_check) { |
301 return clone_loop_predicates(old_entry, new_entry, clone_limit_check, this, &this->_igvn); |
301 return clone_loop_predicates(old_entry, new_entry, clone_limit_check, this, &this->_igvn); |
302 } |
302 } |
303 |
303 |
304 void PhaseIdealLoop::clone_loop_predicates_fix_mem(ProjNode* dom_proj , ProjNode* proj, |
|
305 PhaseIdealLoop* loop_phase, |
|
306 PhaseIterGVN* igvn) { |
|
307 Compile* C = NULL; |
|
308 if (loop_phase != NULL) { |
|
309 igvn = &loop_phase->igvn(); |
|
310 } |
|
311 C = igvn->C; |
|
312 ProjNode* other_dom_proj = dom_proj->in(0)->as_Multi()->proj_out(1-dom_proj->_con); |
|
313 Node* dom_r = other_dom_proj->unique_ctrl_out(); |
|
314 if (dom_r->is_Region()) { |
|
315 assert(dom_r->unique_ctrl_out()->is_Call(), "unc expected"); |
|
316 ProjNode* other_proj = proj->in(0)->as_Multi()->proj_out(1-proj->_con); |
|
317 Node* r = other_proj->unique_ctrl_out(); |
|
318 assert(r->is_Region() && r->unique_ctrl_out()->is_Call(), "cloned predicate should have caused region to be added"); |
|
319 for (DUIterator_Fast imax, i = dom_r->fast_outs(imax); i < imax; i++) { |
|
320 Node* dom_use = dom_r->fast_out(i); |
|
321 if (dom_use->is_Phi() && dom_use->bottom_type() == Type::MEMORY) { |
|
322 assert(dom_use->in(0) == dom_r, ""); |
|
323 Node* phi = NULL; |
|
324 for (DUIterator_Fast jmax, j = r->fast_outs(jmax); j < jmax; j++) { |
|
325 Node* use = r->fast_out(j); |
|
326 if (use->is_Phi() && use->bottom_type() == Type::MEMORY && |
|
327 use->adr_type() == dom_use->adr_type()) { |
|
328 assert(use->in(0) == r, ""); |
|
329 assert(phi == NULL, "only one phi"); |
|
330 phi = use; |
|
331 } |
|
332 } |
|
333 if (phi == NULL) { |
|
334 const TypePtr* adr_type = dom_use->adr_type(); |
|
335 int alias = C->get_alias_index(adr_type); |
|
336 Node* call = r->unique_ctrl_out(); |
|
337 Node* mem = call->in(TypeFunc::Memory); |
|
338 MergeMemNode* mm = NULL; |
|
339 if (mem->is_MergeMem()) { |
|
340 mm = mem->clone()->as_MergeMem(); |
|
341 if (adr_type == TypePtr::BOTTOM) { |
|
342 mem = mem->as_MergeMem()->base_memory(); |
|
343 } else { |
|
344 mem = mem->as_MergeMem()->memory_at(alias); |
|
345 } |
|
346 } else { |
|
347 mm = MergeMemNode::make(mem); |
|
348 } |
|
349 phi = PhiNode::make(r, mem, Type::MEMORY, adr_type); |
|
350 if (adr_type == TypePtr::BOTTOM) { |
|
351 mm->set_base_memory(phi); |
|
352 } else { |
|
353 mm->set_memory_at(alias, phi); |
|
354 } |
|
355 if (loop_phase != NULL) { |
|
356 loop_phase->register_new_node(mm, r); |
|
357 loop_phase->register_new_node(phi, r); |
|
358 } else { |
|
359 igvn->register_new_node_with_optimizer(mm); |
|
360 igvn->register_new_node_with_optimizer(phi); |
|
361 } |
|
362 igvn->replace_input_of(call, TypeFunc::Memory, mm); |
|
363 } |
|
364 igvn->replace_input_of(phi, r->find_edge(other_proj), dom_use->in(dom_r->find_edge(other_dom_proj))); |
|
365 } |
|
366 } |
|
367 } |
|
368 } |
|
369 |
|
370 |
|
371 // Clone loop predicates to cloned loops (peeled, unswitched, split_if). |
304 // Clone loop predicates to cloned loops (peeled, unswitched, split_if). |
372 Node* PhaseIdealLoop::clone_loop_predicates(Node* old_entry, Node* new_entry, |
305 Node* PhaseIdealLoop::clone_loop_predicates(Node* old_entry, Node* new_entry, |
373 bool clone_limit_check, |
306 bool clone_limit_check, |
374 PhaseIdealLoop* loop_phase, |
307 PhaseIdealLoop* loop_phase, |
375 PhaseIterGVN* igvn) { |
308 PhaseIterGVN* igvn) { |
398 if (UseLoopPredicate) { |
331 if (UseLoopPredicate) { |
399 predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); |
332 predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); |
400 } |
333 } |
401 if (predicate_proj != NULL) { // right pattern that can be used by loop predication |
334 if (predicate_proj != NULL) { // right pattern that can be used by loop predication |
402 // clone predicate |
335 // clone predicate |
403 ProjNode* proj = clone_predicate(predicate_proj, new_entry, |
336 new_entry = clone_predicate(predicate_proj, new_entry, |
404 Deoptimization::Reason_predicate, |
337 Deoptimization::Reason_predicate, |
405 loop_phase, igvn); |
338 loop_phase, igvn); |
406 assert(proj != NULL, "IfTrue or IfFalse after clone predicate"); |
339 assert(new_entry != NULL && new_entry->is_Proj(), "IfTrue or IfFalse after clone predicate"); |
407 new_entry = proj; |
|
408 if (TraceLoopPredicate) { |
340 if (TraceLoopPredicate) { |
409 tty->print("Loop Predicate cloned: "); |
341 tty->print("Loop Predicate cloned: "); |
410 debug_only( new_entry->in(0)->dump(); ); |
342 debug_only( new_entry->in(0)->dump(); ); |
411 } |
|
412 if (profile_predicate_proj != NULL) { |
|
413 // A node that produces memory may be out of loop and depend on |
|
414 // a profiled predicates. In that case the memory state at the |
|
415 // end of profiled predicates and at the end of predicates are |
|
416 // not the same. The cloned predicates are dominated by the |
|
417 // profiled predicates but may have the wrong memory |
|
418 // state. Update it. |
|
419 clone_loop_predicates_fix_mem(profile_predicate_proj, proj, loop_phase, igvn); |
|
420 } |
343 } |
421 } |
344 } |
422 if (profile_predicate_proj != NULL) { // right pattern that can be used by loop predication |
345 if (profile_predicate_proj != NULL) { // right pattern that can be used by loop predication |
423 // clone predicate |
346 // clone predicate |
424 new_entry = clone_predicate(profile_predicate_proj, new_entry, |
347 new_entry = clone_predicate(profile_predicate_proj, new_entry, |
1456 hoisted = loop_predication_impl_helper(loop, proj, predicate_proj, cl, zero, invar, Deoptimization::Reason_predicate) | hoisted; |
1379 hoisted = loop_predication_impl_helper(loop, proj, predicate_proj, cl, zero, invar, Deoptimization::Reason_predicate) | hoisted; |
1457 } |
1380 } |
1458 } // end while |
1381 } // end while |
1459 } |
1382 } |
1460 |
1383 |
1461 Node_List if_proj_list_freq(area); |
|
1462 if (follow_branches) { |
1384 if (follow_branches) { |
1463 PathFrequency pf(loop->_head, this); |
1385 PathFrequency pf(loop->_head, this); |
1464 |
1386 |
1465 // Some projections were skipped by regular predicates because of |
1387 // Some projections were skipped by regular predicates because of |
1466 // an early loop exit. Try them with profile data. |
1388 // an early loop exit. Try them with profile data. |
1474 } |
1396 } |
1475 |
1397 |
1476 // And look into all branches |
1398 // And look into all branches |
1477 Node_Stack stack(0); |
1399 Node_Stack stack(0); |
1478 VectorSet seen(Thread::current()->resource_area()); |
1400 VectorSet seen(Thread::current()->resource_area()); |
|
1401 Node_List if_proj_list_freq(area); |
1479 while (regions.size() > 0) { |
1402 while (regions.size() > 0) { |
1480 Node* c = regions.pop(); |
1403 Node* c = regions.pop(); |
1481 loop_predication_follow_branches(c, loop, loop_trip_cnt, pf, stack, seen, if_proj_list_freq); |
1404 loop_predication_follow_branches(c, loop, loop_trip_cnt, pf, stack, seen, if_proj_list_freq); |
1482 } |
1405 } |
1483 |
1406 |