299 // Interface from PhaseIdealLoop |
299 // Interface from PhaseIdealLoop |
300 Node* PhaseIdealLoop::clone_loop_predicates(Node* old_entry, Node* new_entry, bool clone_limit_check) { |
300 Node* PhaseIdealLoop::clone_loop_predicates(Node* old_entry, Node* new_entry, bool clone_limit_check) { |
301 return clone_loop_predicates(old_entry, new_entry, clone_limit_check, this, &this->_igvn); |
301 return clone_loop_predicates(old_entry, new_entry, clone_limit_check, this, &this->_igvn); |
302 } |
302 } |
303 |
303 |
|
304 void PhaseIdealLoop::clone_loop_predicates_fix_mem(ProjNode* dom_proj , ProjNode* proj, |
|
305 PhaseIdealLoop* loop_phase, |
|
306 PhaseIterGVN* igvn) { |
|
307 Compile* C = NULL; |
|
308 if (loop_phase != NULL) { |
|
309 igvn = &loop_phase->igvn(); |
|
310 } |
|
311 C = igvn->C; |
|
312 ProjNode* other_dom_proj = dom_proj->in(0)->as_Multi()->proj_out(1-dom_proj->_con); |
|
313 Node* dom_r = other_dom_proj->unique_ctrl_out(); |
|
314 if (dom_r->is_Region()) { |
|
315 assert(dom_r->unique_ctrl_out()->is_Call(), "unc expected"); |
|
316 ProjNode* other_proj = proj->in(0)->as_Multi()->proj_out(1-proj->_con); |
|
317 Node* r = other_proj->unique_ctrl_out(); |
|
318 assert(r->is_Region() && r->unique_ctrl_out()->is_Call(), "cloned predicate should have caused region to be added"); |
|
319 for (DUIterator_Fast imax, i = dom_r->fast_outs(imax); i < imax; i++) { |
|
320 Node* dom_use = dom_r->fast_out(i); |
|
321 if (dom_use->is_Phi() && dom_use->bottom_type() == Type::MEMORY) { |
|
322 assert(dom_use->in(0) == dom_r, ""); |
|
323 Node* phi = NULL; |
|
324 for (DUIterator_Fast jmax, j = r->fast_outs(jmax); j < jmax; j++) { |
|
325 Node* use = r->fast_out(j); |
|
326 if (use->is_Phi() && use->bottom_type() == Type::MEMORY && |
|
327 use->adr_type() == dom_use->adr_type()) { |
|
328 assert(use->in(0) == r, ""); |
|
329 assert(phi == NULL, "only one phi"); |
|
330 phi = use; |
|
331 } |
|
332 } |
|
333 if (phi == NULL) { |
|
334 const TypePtr* adr_type = dom_use->adr_type(); |
|
335 int alias = C->get_alias_index(adr_type); |
|
336 Node* call = r->unique_ctrl_out(); |
|
337 Node* mem = call->in(TypeFunc::Memory); |
|
338 MergeMemNode* mm = NULL; |
|
339 if (mem->is_MergeMem()) { |
|
340 mm = mem->clone()->as_MergeMem(); |
|
341 if (adr_type == TypePtr::BOTTOM) { |
|
342 mem = mem->as_MergeMem()->base_memory(); |
|
343 } else { |
|
344 mem = mem->as_MergeMem()->memory_at(alias); |
|
345 } |
|
346 } else { |
|
347 mm = MergeMemNode::make(mem); |
|
348 } |
|
349 phi = PhiNode::make(r, mem, Type::MEMORY, adr_type); |
|
350 if (adr_type == TypePtr::BOTTOM) { |
|
351 mm->set_base_memory(phi); |
|
352 } else { |
|
353 mm->set_memory_at(alias, phi); |
|
354 } |
|
355 if (loop_phase != NULL) { |
|
356 loop_phase->register_new_node(mm, r); |
|
357 loop_phase->register_new_node(phi, r); |
|
358 } else { |
|
359 igvn->register_new_node_with_optimizer(mm); |
|
360 igvn->register_new_node_with_optimizer(phi); |
|
361 } |
|
362 igvn->replace_input_of(call, TypeFunc::Memory, mm); |
|
363 } |
|
364 igvn->replace_input_of(phi, r->find_edge(other_proj), dom_use->in(dom_r->find_edge(other_dom_proj))); |
|
365 } |
|
366 } |
|
367 } |
|
368 } |
|
369 |
|
370 |
304 // Clone loop predicates to cloned loops (peeled, unswitched, split_if). |
371 // Clone loop predicates to cloned loops (peeled, unswitched, split_if). |
305 Node* PhaseIdealLoop::clone_loop_predicates(Node* old_entry, Node* new_entry, |
372 Node* PhaseIdealLoop::clone_loop_predicates(Node* old_entry, Node* new_entry, |
306 bool clone_limit_check, |
373 bool clone_limit_check, |
307 PhaseIdealLoop* loop_phase, |
374 PhaseIdealLoop* loop_phase, |
308 PhaseIterGVN* igvn) { |
375 PhaseIterGVN* igvn) { |
331 if (UseLoopPredicate) { |
398 if (UseLoopPredicate) { |
332 predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); |
399 predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); |
333 } |
400 } |
334 if (predicate_proj != NULL) { // right pattern that can be used by loop predication |
401 if (predicate_proj != NULL) { // right pattern that can be used by loop predication |
335 // clone predicate |
402 // clone predicate |
336 new_entry = clone_predicate(predicate_proj, new_entry, |
403 ProjNode* proj = clone_predicate(predicate_proj, new_entry, |
337 Deoptimization::Reason_predicate, |
404 Deoptimization::Reason_predicate, |
338 loop_phase, igvn); |
405 loop_phase, igvn); |
339 assert(new_entry != NULL && new_entry->is_Proj(), "IfTrue or IfFalse after clone predicate"); |
406 assert(proj != NULL, "IfTrue or IfFalse after clone predicate"); |
|
407 new_entry = proj; |
340 if (TraceLoopPredicate) { |
408 if (TraceLoopPredicate) { |
341 tty->print("Loop Predicate cloned: "); |
409 tty->print("Loop Predicate cloned: "); |
342 debug_only( new_entry->in(0)->dump(); ); |
410 debug_only( new_entry->in(0)->dump(); ); |
|
411 } |
|
412 if (profile_predicate_proj != NULL) { |
|
413 // A node that produces memory may be out of loop and depend on |
|
414 // a profiled predicates. In that case the memory state at the |
|
415 // end of profiled predicates and at the end of predicates are |
|
416 // not the same. The cloned predicates are dominated by the |
|
417 // profiled predicates but may have the wrong memory |
|
418 // state. Update it. |
|
419 clone_loop_predicates_fix_mem(profile_predicate_proj, proj, loop_phase, igvn); |
343 } |
420 } |
344 } |
421 } |
345 if (profile_predicate_proj != NULL) { // right pattern that can be used by loop predication |
422 if (profile_predicate_proj != NULL) { // right pattern that can be used by loop predication |
346 // clone predicate |
423 // clone predicate |
347 new_entry = clone_predicate(profile_predicate_proj, new_entry, |
424 new_entry = clone_predicate(profile_predicate_proj, new_entry, |