2657 wb->pin_and_expand_move_barrier(phase, memory_graph_fixers, uses); |
2657 wb->pin_and_expand_move_barrier(phase, memory_graph_fixers, uses); |
2658 |
2658 |
2659 wb->pin_and_expand_helper(phase); |
2659 wb->pin_and_expand_helper(phase); |
2660 } |
2660 } |
2661 |
2661 |
|
2662 for (uint i = 0; i < enqueue_barriers.size(); i++) { |
|
2663 Node* barrier = enqueue_barriers.at(i); |
|
2664 Node* ctrl = phase->get_ctrl(barrier); |
|
2665 IdealLoopTree* loop = phase->get_loop(ctrl); |
|
2666 if (loop->_head->is_OuterStripMinedLoop()) { |
|
2667 // Expanding a barrier here will break loop strip mining |
|
2668 // verification. Transform the loop so the loop nest doesn't |
|
2669 // appear as strip mined. |
|
2670 OuterStripMinedLoopNode* outer = loop->_head->as_OuterStripMinedLoop(); |
|
2671 hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase); |
|
2672 } |
|
2673 } |
|
2674 |
|
2675 for (int i = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i > 0; i--) { |
|
2676 int cnt = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); |
|
2677 ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i-1); |
|
2678 Node* ctrl = phase->get_ctrl(wb); |
|
2679 IdealLoopTree* loop = phase->get_loop(ctrl); |
|
2680 if (loop->_head->is_OuterStripMinedLoop()) { |
|
2681 // Expanding a barrier here will break loop strip mining |
|
2682 // verification. Transform the loop so the loop nest doesn't |
|
2683 // appear as strip mined. |
|
2684 OuterStripMinedLoopNode* outer = loop->_head->as_OuterStripMinedLoop(); |
|
2685 hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase); |
|
2686 } |
|
2687 } |
|
2688 |
2662 MemoryGraphFixer fixer(Compile::AliasIdxRaw, true, phase); |
2689 MemoryGraphFixer fixer(Compile::AliasIdxRaw, true, phase); |
2663 Unique_Node_List uses_to_ignore; |
2690 Unique_Node_List uses_to_ignore; |
2664 Unique_Node_List outer_lsms; |
|
2665 for (uint i = 0; i < enqueue_barriers.size(); i++) { |
2691 for (uint i = 0; i < enqueue_barriers.size(); i++) { |
2666 Node* barrier = enqueue_barriers.at(i); |
2692 Node* barrier = enqueue_barriers.at(i); |
2667 Node* pre_val = barrier->in(1); |
2693 Node* pre_val = barrier->in(1); |
2668 |
2694 |
2669 if (phase->igvn().type(pre_val)->higher_equal(TypePtr::NULL_PTR)) { |
2695 if (phase->igvn().type(pre_val)->higher_equal(TypePtr::NULL_PTR)) { |
2683 phase->set_ctrl(barrier, ctrl); |
2709 phase->set_ctrl(barrier, ctrl); |
2684 } |
2710 } |
2685 |
2711 |
2686 Node* init_ctrl = ctrl; |
2712 Node* init_ctrl = ctrl; |
2687 IdealLoopTree* loop = phase->get_loop(ctrl); |
2713 IdealLoopTree* loop = phase->get_loop(ctrl); |
2688 if (loop->_head->is_OuterStripMinedLoop()) { |
|
2689 outer_lsms.push(loop->_head); |
|
2690 } |
|
2691 Node* raw_mem = fixer.find_mem(ctrl, barrier); |
2714 Node* raw_mem = fixer.find_mem(ctrl, barrier); |
2692 Node* init_raw_mem = raw_mem; |
2715 Node* init_raw_mem = raw_mem; |
2693 Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL); |
2716 Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL); |
2694 Node* heap_stable_ctrl = NULL; |
2717 Node* heap_stable_ctrl = NULL; |
2695 Node* null_ctrl = NULL; |
2718 Node* null_ctrl = NULL; |
2830 Node* init_wb_mem = wb_mem; |
2853 Node* init_wb_mem = wb_mem; |
2831 |
2854 |
2832 Node* val = wb->in(ValueIn); |
2855 Node* val = wb->in(ValueIn); |
2833 Node* wbproj = wb->find_out_with(Op_ShenandoahWBMemProj); |
2856 Node* wbproj = wb->find_out_with(Op_ShenandoahWBMemProj); |
2834 IdealLoopTree *loop = phase->get_loop(ctrl); |
2857 IdealLoopTree *loop = phase->get_loop(ctrl); |
2835 if (loop->_head->is_OuterStripMinedLoop()) { |
|
2836 outer_lsms.push(loop->_head); |
|
2837 } |
|
2838 |
2858 |
2839 assert(val->Opcode() != Op_ShenandoahWriteBarrier, "No chain of write barriers"); |
2859 assert(val->Opcode() != Op_ShenandoahWriteBarrier, "No chain of write barriers"); |
2840 |
2860 |
2841 CallStaticJavaNode* unc = wb->pin_and_expand_null_check(phase->igvn()); |
2861 CallStaticJavaNode* unc = wb->pin_and_expand_null_check(phase->igvn()); |
2842 Node* unc_ctrl = NULL; |
2862 Node* unc_ctrl = NULL; |
3017 fixer.fix_mem(ctrl, region, init_raw_mem, raw_mem_for_ctrl, raw_mem_phi, uses); |
3037 fixer.fix_mem(ctrl, region, init_raw_mem, raw_mem_for_ctrl, raw_mem_phi, uses); |
3018 assert(ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() == cnt - 1, "not replaced"); |
3038 assert(ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() == cnt - 1, "not replaced"); |
3019 } |
3039 } |
3020 |
3040 |
3021 assert(ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() == 0, "all write barrier nodes should have been replaced"); |
3041 assert(ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() == 0, "all write barrier nodes should have been replaced"); |
3022 |
|
3023 for (uint i = 0; i < outer_lsms.size(); i++) { |
|
3024 // Expanding a barrier here will break loop strip mining |
|
3025 // verification. Transform the loop so the loop nest doesn't |
|
3026 // appear as strip mined. |
|
3027 OuterStripMinedLoopNode* outer = outer_lsms.at(i)->as_OuterStripMinedLoop(); |
|
3028 hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase); |
|
3029 } |
|
3030 } |
3042 } |
3031 |
3043 |
3032 void ShenandoahWriteBarrierNode::move_heap_stable_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) { |
3044 void ShenandoahWriteBarrierNode::move_heap_stable_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) { |
3033 IdealLoopTree *loop = phase->get_loop(iff); |
3045 IdealLoopTree *loop = phase->get_loop(iff); |
3034 Node* loop_head = loop->_head; |
3046 Node* loop_head = loop->_head; |