206 } |
206 } |
207 } else if (in->Opcode() == Op_LoadP && in->adr_type() == TypeRawPtr::BOTTOM) { |
207 } else if (in->Opcode() == Op_LoadP && in->adr_type() == TypeRawPtr::BOTTOM) { |
208 if (trace) { |
208 if (trace) { |
209 tty->print("Found raw LoadP (OSR argument?)"); |
209 tty->print("Found raw LoadP (OSR argument?)"); |
210 } |
210 } |
211 } else if (in->Opcode() == Op_ShenandoahLoadReferenceBarrier || |
211 } else if (in->Opcode() == Op_ShenandoahLoadReferenceBarrier) { |
212 (in->Opcode() == Op_Proj && |
|
213 in->in(0)->Opcode() == Op_CallLeaf && |
|
214 strcmp(in->in(0)->as_Call()->_name, "ShenandoahRuntime::load_reference_barrier_native") == 0)) { |
|
215 if (t == ShenandoahOopStore) { |
212 if (t == ShenandoahOopStore) { |
216 uint i = 0; |
213 uint i = 0; |
217 for (; i < phis.size(); i++) { |
214 for (; i < phis.size(); i++) { |
218 Node* n = phis.node_at(i); |
215 Node* n = phis.node_at(i); |
219 if (n->Opcode() == Op_ShenandoahEnqueueBarrier) { |
216 if (n->Opcode() == Op_ShenandoahEnqueueBarrier) { |
528 } else if (strlen(call->_name) > 5 && |
525 } else if (strlen(call->_name) > 5 && |
529 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) { |
526 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) { |
530 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) { |
527 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) { |
531 report_verify_failure("Shenandoah verification: _fill should have barriers", n); |
528 report_verify_failure("Shenandoah verification: _fill should have barriers", n); |
532 } |
529 } |
533 } else if (!strcmp(call->_name, "shenandoah_wb_pre") || !strcmp(call->_name, "ShenandoahRuntime::load_reference_barrier_native")) { |
530 } else if (!strcmp(call->_name, "shenandoah_wb_pre")) { |
534 // skip |
531 // skip |
535 } else { |
532 } else { |
536 const int calls_len = sizeof(calls) / sizeof(calls[0]); |
533 const int calls_len = sizeof(calls) / sizeof(calls[0]); |
537 int i = 0; |
534 int i = 0; |
538 for (; i < calls_len; i++) { |
535 for (; i < calls_len; i++) { |
1083 |
1080 |
1084 ctrl = new IfFalseNode(in_cset_fast_test_iff); |
1081 ctrl = new IfFalseNode(in_cset_fast_test_iff); |
1085 phase->register_control(ctrl, loop, in_cset_fast_test_iff); |
1082 phase->register_control(ctrl, loop, in_cset_fast_test_iff); |
1086 } |
1083 } |
1087 |
1084 |
1088 void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node*& result_mem, Node* raw_mem, PhaseIdealLoop* phase) { |
1085 void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node*& result_mem, Node* raw_mem, bool is_native, PhaseIdealLoop* phase) { |
1089 IdealLoopTree*loop = phase->get_loop(ctrl); |
1086 IdealLoopTree*loop = phase->get_loop(ctrl); |
1090 const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst(); |
1087 const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst(); |
1091 |
1088 |
1092 // The slow path stub consumes and produces raw memory in addition |
1089 // The slow path stub consumes and produces raw memory in addition |
1093 // to the existing memory edges |
1090 // to the existing memory edges |
1094 Node* base = find_bottom_mem(ctrl, phase); |
1091 Node* base = find_bottom_mem(ctrl, phase); |
1095 MergeMemNode* mm = MergeMemNode::make(base); |
1092 MergeMemNode* mm = MergeMemNode::make(base); |
1096 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem); |
1093 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem); |
1097 phase->register_new_node(mm, ctrl); |
1094 phase->register_new_node(mm, ctrl); |
1098 |
1095 |
1099 Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(), |
1096 address calladdr = is_native ? CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native) |
1100 CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), |
1097 : CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier); |
1101 "shenandoah_load_reference_barrier", TypeRawPtr::BOTTOM); |
1098 const char* name = is_native ? "oop_load_from_native_barrier" : "load_reference_barrier"; |
|
1099 Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(), calladdr, name, TypeRawPtr::BOTTOM); |
1102 call->init_req(TypeFunc::Control, ctrl); |
1100 call->init_req(TypeFunc::Control, ctrl); |
1103 call->init_req(TypeFunc::I_O, phase->C->top()); |
1101 call->init_req(TypeFunc::I_O, phase->C->top()); |
1104 call->init_req(TypeFunc::Memory, mm); |
1102 call->init_req(TypeFunc::Memory, mm); |
1105 call->init_req(TypeFunc::FramePtr, phase->C->top()); |
1103 call->init_req(TypeFunc::FramePtr, phase->C->top()); |
1106 call->init_req(TypeFunc::ReturnAdr, phase->C->top()); |
1104 call->init_req(TypeFunc::ReturnAdr, phase->C->top()); |
1559 |
1557 |
1560 // Call lrb-stub and wire up that path in slots 4 |
1558 // Call lrb-stub and wire up that path in slots 4 |
1561 Node* result_mem = NULL; |
1559 Node* result_mem = NULL; |
1562 ctrl = if_not_fwd; |
1560 ctrl = if_not_fwd; |
1563 fwd = new_val; |
1561 fwd = new_val; |
1564 call_lrb_stub(ctrl, fwd, result_mem, raw_mem, phase); |
1562 call_lrb_stub(ctrl, fwd, result_mem, raw_mem, lrb->is_native(), phase); |
1565 region->init_req(_evac_path, ctrl); |
1563 region->init_req(_evac_path, ctrl); |
1566 val_phi->init_req(_evac_path, fwd); |
1564 val_phi->init_req(_evac_path, fwd); |
1567 raw_mem_phi->init_req(_evac_path, result_mem); |
1565 raw_mem_phi->init_req(_evac_path, result_mem); |
1568 |
1566 |
1569 phase->register_control(region, loop, heap_stable_iff); |
1567 phase->register_control(region, loop, heap_stable_iff); |
3002 } |
3000 } |
3003 } |
3001 } |
3004 } |
3002 } |
3005 } |
3003 } |
3006 |
3004 |
3007 ShenandoahLoadReferenceBarrierNode::ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* obj) |
3005 ShenandoahLoadReferenceBarrierNode::ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* obj, bool native) |
3008 : Node(ctrl, obj) { |
3006 : Node(ctrl, obj), _native(native) { |
3009 ShenandoahBarrierSetC2::bsc2()->state()->add_load_reference_barrier(this); |
3007 ShenandoahBarrierSetC2::bsc2()->state()->add_load_reference_barrier(this); |
|
3008 } |
|
3009 |
|
3010 bool ShenandoahLoadReferenceBarrierNode::is_native() const { |
|
3011 return _native; |
|
3012 } |
|
3013 |
|
3014 uint ShenandoahLoadReferenceBarrierNode::size_of() const { |
|
3015 return sizeof(*this); |
|
3016 } |
|
3017 |
|
3018 uint ShenandoahLoadReferenceBarrierNode::hash() const { |
|
3019 return Node::hash() + _native ? 1 : 0; |
|
3020 } |
|
3021 |
|
3022 bool ShenandoahLoadReferenceBarrierNode::cmp( const Node &n ) const { |
|
3023 return Node::cmp(n) && n.Opcode() == Op_ShenandoahLoadReferenceBarrier && |
|
3024 _native == ((const ShenandoahLoadReferenceBarrierNode&)n)._native; |
3010 } |
3025 } |
3011 |
3026 |
3012 const Type* ShenandoahLoadReferenceBarrierNode::bottom_type() const { |
3027 const Type* ShenandoahLoadReferenceBarrierNode::bottom_type() const { |
3013 if (in(ValueIn) == NULL || in(ValueIn)->is_top()) { |
3028 if (in(ValueIn) == NULL || in(ValueIn)->is_top()) { |
3014 return Type::TOP; |
3029 return Type::TOP; |