321 if (trace) {tty->print_cr("Mark load");} |
317 if (trace) {tty->print_cr("Mark load");} |
322 } else if (adr_type->isa_instptr() && |
318 } else if (adr_type->isa_instptr() && |
323 adr_type->is_instptr()->klass()->is_subtype_of(Compile::current()->env()->Reference_klass()) && |
319 adr_type->is_instptr()->klass()->is_subtype_of(Compile::current()->env()->Reference_klass()) && |
324 adr_type->is_instptr()->offset() == java_lang_ref_Reference::referent_offset) { |
320 adr_type->is_instptr()->offset() == java_lang_ref_Reference::referent_offset) { |
325 if (trace) {tty->print_cr("Reference.get()");} |
321 if (trace) {tty->print_cr("Reference.get()");} |
326 } else { |
322 } else if (!verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahLoad, trace, barriers_used)) { |
327 bool verify = true; |
323 report_verify_failure("Shenandoah verification: Load should have barriers", n); |
328 if (adr_type->isa_instptr()) { |
|
329 const TypeInstPtr* tinst = adr_type->is_instptr(); |
|
330 ciKlass* k = tinst->klass(); |
|
331 assert(k->is_instance_klass(), ""); |
|
332 ciInstanceKlass* ik = (ciInstanceKlass*)k; |
|
333 int offset = adr_type->offset(); |
|
334 |
|
335 if ((ik->debug_final_field_at(offset) && ShenandoahOptimizeInstanceFinals) || |
|
336 (ik->debug_stable_field_at(offset) && ShenandoahOptimizeStableFinals)) { |
|
337 if (trace) {tty->print_cr("Final/stable");} |
|
338 verify = false; |
|
339 } else if (k == ciEnv::current()->Class_klass() && |
|
340 tinst->const_oop() != NULL && |
|
341 tinst->offset() >= (ik->size_helper() * wordSize)) { |
|
342 ciInstanceKlass* k = tinst->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass(); |
|
343 ciField* field = k->get_field_by_offset(tinst->offset(), true); |
|
344 if ((ShenandoahOptimizeStaticFinals && field->is_final()) || |
|
345 (ShenandoahOptimizeStableFinals && field->is_stable())) { |
|
346 verify = false; |
|
347 } |
|
348 } |
|
349 } |
|
350 |
|
351 if (verify && !verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahLoad, trace, barriers_used)) { |
|
352 report_verify_failure("Shenandoah verification: Load should have barriers", n); |
|
353 } |
|
354 } |
324 } |
355 } |
325 } |
356 } else if (n->is_Store()) { |
326 } else if (n->is_Store()) { |
357 const bool trace = false; |
327 const bool trace = false; |
358 |
328 |
668 visited.Reset(); |
638 visited.Reset(); |
669 } |
639 } |
670 } |
640 } |
671 } |
641 } |
672 } |
642 } |
673 for( uint i = 0; i < n->len(); ++i ) { |
|
674 Node *m = n->in(i); |
|
675 if (m == NULL) continue; |
|
676 |
|
677 // In most cases, inputs should be known to be non null. If it's |
|
678 // not the case, it could be a missing cast_not_null() in an |
|
679 // intrinsic or support might be needed in AddPNode::Ideal() to |
|
680 // avoid a NULL+offset input. |
|
681 if (!(n->is_Phi() || |
|
682 (n->is_SafePoint() && (!n->is_CallRuntime() || !strcmp(n->as_Call()->_name, "shenandoah_wb_pre") || !strcmp(n->as_Call()->_name, "unsafe_arraycopy"))) || |
|
683 n->Opcode() == Op_CmpP || |
|
684 n->Opcode() == Op_CmpN || |
|
685 (n->Opcode() == Op_StoreP && i == StoreNode::ValueIn) || |
|
686 (n->Opcode() == Op_StoreN && i == StoreNode::ValueIn) || |
|
687 n->is_ConstraintCast() || |
|
688 n->Opcode() == Op_Return || |
|
689 n->Opcode() == Op_Conv2B || |
|
690 n->is_AddP() || |
|
691 n->Opcode() == Op_CMoveP || |
|
692 n->Opcode() == Op_CMoveN || |
|
693 n->Opcode() == Op_Rethrow || |
|
694 n->is_MemBar() || |
|
695 n->is_Mem() || |
|
696 n->Opcode() == Op_AryEq || |
|
697 n->Opcode() == Op_SCMemProj || |
|
698 n->Opcode() == Op_EncodeP || |
|
699 n->Opcode() == Op_DecodeN || |
|
700 n->Opcode() == Op_ShenandoahEnqueueBarrier || |
|
701 n->Opcode() == Op_ShenandoahLoadReferenceBarrier)) { |
|
702 if (m->bottom_type()->make_oopptr() && m->bottom_type()->make_oopptr()->meet(TypePtr::NULL_PTR) == m->bottom_type()) { |
|
703 report_verify_failure("Shenandoah verification: null input", n, m); |
|
704 } |
|
705 } |
|
706 |
|
707 wq.push(m); |
|
708 } |
|
709 } |
643 } |
710 |
644 |
711 if (verify_no_useless_barrier) { |
645 if (verify_no_useless_barrier) { |
712 for (int i = 0; i < barriers.length(); i++) { |
646 for (int i = 0; i < barriers.length(); i++) { |
713 Node* n = barriers.at(i); |
647 Node* n = barriers.at(i); |
1080 |
1014 |
1081 ctrl = new IfFalseNode(in_cset_fast_test_iff); |
1015 ctrl = new IfFalseNode(in_cset_fast_test_iff); |
1082 phase->register_control(ctrl, loop, in_cset_fast_test_iff); |
1016 phase->register_control(ctrl, loop, in_cset_fast_test_iff); |
1083 } |
1017 } |
1084 |
1018 |
1085 void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node*& result_mem, Node* raw_mem, PhaseIdealLoop* phase) { |
1019 void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem, bool is_native, PhaseIdealLoop* phase) { |
1086 IdealLoopTree*loop = phase->get_loop(ctrl); |
1020 IdealLoopTree*loop = phase->get_loop(ctrl); |
1087 const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst(); |
1021 const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst(); |
1088 |
1022 |
1089 // The slow path stub consumes and produces raw memory in addition |
1023 // The slow path stub consumes and produces raw memory in addition |
1090 // to the existing memory edges |
1024 // to the existing memory edges |
1091 Node* base = find_bottom_mem(ctrl, phase); |
1025 Node* base = find_bottom_mem(ctrl, phase); |
1092 MergeMemNode* mm = MergeMemNode::make(base); |
1026 MergeMemNode* mm = MergeMemNode::make(base); |
1093 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem); |
1027 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem); |
1094 phase->register_new_node(mm, ctrl); |
1028 phase->register_new_node(mm, ctrl); |
1095 |
1029 |
1096 Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT), "shenandoah_load_reference_barrier", TypeRawPtr::BOTTOM); |
1030 address target = LP64_ONLY(UseCompressedOops) NOT_LP64(false) ? |
|
1031 CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup_narrow) : |
|
1032 CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup); |
|
1033 |
|
1034 address calladdr = is_native ? CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native) |
|
1035 : target; |
|
1036 const char* name = is_native ? "oop_load_from_native_barrier" : "load_reference_barrier"; |
|
1037 Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(), calladdr, name, TypeRawPtr::BOTTOM); |
|
1038 |
1097 call->init_req(TypeFunc::Control, ctrl); |
1039 call->init_req(TypeFunc::Control, ctrl); |
1098 call->init_req(TypeFunc::I_O, phase->C->top()); |
1040 call->init_req(TypeFunc::I_O, phase->C->top()); |
1099 call->init_req(TypeFunc::Memory, mm); |
1041 call->init_req(TypeFunc::Memory, mm); |
1100 call->init_req(TypeFunc::FramePtr, phase->C->top()); |
1042 call->init_req(TypeFunc::FramePtr, phase->C->top()); |
1101 call->init_req(TypeFunc::ReturnAdr, phase->C->top()); |
1043 call->init_req(TypeFunc::ReturnAdr, phase->C->top()); |
1102 call->init_req(TypeFunc::Parms, val); |
1044 call->init_req(TypeFunc::Parms, val); |
|
1045 call->init_req(TypeFunc::Parms+1, load_addr); |
1103 phase->register_control(call, loop, ctrl); |
1046 phase->register_control(call, loop, ctrl); |
1104 ctrl = new ProjNode(call, TypeFunc::Control); |
1047 ctrl = new ProjNode(call, TypeFunc::Control); |
1105 phase->register_control(ctrl, loop, call); |
1048 phase->register_control(ctrl, loop, call); |
1106 result_mem = new ProjNode(call, TypeFunc::Memory); |
1049 result_mem = new ProjNode(call, TypeFunc::Memory); |
1107 phase->register_new_node(result_mem, call); |
1050 phase->register_new_node(result_mem, call); |
1327 } |
1270 } |
1328 } |
1271 } |
1329 } |
1272 } |
1330 if ((ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) || ctrl->is_CallJava()) { |
1273 if ((ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) || ctrl->is_CallJava()) { |
1331 CallNode* call = ctrl->is_Proj() ? ctrl->in(0)->as_CallJava() : ctrl->as_CallJava(); |
1274 CallNode* call = ctrl->is_Proj() ? ctrl->in(0)->as_CallJava() : ctrl->as_CallJava(); |
|
1275 if (call->entry_point() == OptoRuntime::rethrow_stub()) { |
|
1276 // The rethrow call may have too many projections to be |
|
1277 // properly handled here. Given there's no reason for a |
|
1278 // barrier to depend on the call, move it above the call |
|
1279 stack.push(lrb, 0); |
|
1280 do { |
|
1281 Node* n = stack.node(); |
|
1282 uint idx = stack.index(); |
|
1283 if (idx < n->req()) { |
|
1284 Node* in = n->in(idx); |
|
1285 stack.set_index(idx+1); |
|
1286 if (in != NULL) { |
|
1287 if (phase->has_ctrl(in)) { |
|
1288 if (phase->is_dominator(call, phase->get_ctrl(in))) { |
|
1289 #ifdef ASSERT |
|
1290 for (uint i = 0; i < stack.size(); i++) { |
|
1291 assert(stack.node_at(i) != in, "node shouldn't have been seen yet"); |
|
1292 } |
|
1293 #endif |
|
1294 stack.push(in, 0); |
|
1295 } |
|
1296 } else { |
|
1297 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call"); |
|
1298 } |
|
1299 } |
|
1300 } else { |
|
1301 phase->set_ctrl(n, call->in(0)); |
|
1302 stack.pop(); |
|
1303 } |
|
1304 } while(stack.size() > 0); |
|
1305 continue; |
|
1306 } |
1332 CallProjections projs; |
1307 CallProjections projs; |
1333 call->extract_projections(&projs, false, false); |
1308 call->extract_projections(&projs, false, false); |
1334 |
1309 |
1335 Node* lrb_clone = lrb->clone(); |
1310 Node* lrb_clone = lrb->clone(); |
1336 phase->register_new_node(lrb_clone, projs.catchall_catchproj); |
1311 phase->register_new_node(lrb_clone, projs.catchall_catchproj); |
1462 Node* null_ctrl = NULL; |
1437 Node* null_ctrl = NULL; |
1463 |
1438 |
1464 assert(val->bottom_type()->make_oopptr(), "need oop"); |
1439 assert(val->bottom_type()->make_oopptr(), "need oop"); |
1465 assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant"); |
1440 assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant"); |
1466 |
1441 |
1467 enum { _heap_stable = 1, _not_cset, _fwded, _evac_path, _null_path, PATH_LIMIT }; |
1442 enum { _heap_stable = 1, _not_cset, _evac_path, _null_path, PATH_LIMIT }; |
1468 Node* region = new RegionNode(PATH_LIMIT); |
1443 Node* region = new RegionNode(PATH_LIMIT); |
1469 Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr()); |
1444 Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr()); |
1470 Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); |
1445 Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); |
1471 |
1446 |
1472 // Stable path. |
1447 // Stable path. |
1512 fix_null_check(unc, unc_ctrl, ctrl->in(0)->as_If()->proj_out(0), uses, phase); |
1487 fix_null_check(unc, unc_ctrl, ctrl->in(0)->as_If()->proj_out(0), uses, phase); |
1513 |
1488 |
1514 IfNode* iff = unc_ctrl->in(0)->as_If(); |
1489 IfNode* iff = unc_ctrl->in(0)->as_If(); |
1515 phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1)); |
1490 phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1)); |
1516 } |
1491 } |
1517 Node* addr = new AddPNode(new_val, uncasted_val, phase->igvn().MakeConX(oopDesc::mark_offset_in_bytes())); |
|
1518 phase->register_new_node(addr, ctrl); |
|
1519 assert(new_val->bottom_type()->isa_oopptr(), "what else?"); |
|
1520 Node* markword = new LoadXNode(ctrl, raw_mem, addr, TypeRawPtr::BOTTOM, TypeX_X, MemNode::unordered); |
|
1521 phase->register_new_node(markword, ctrl); |
|
1522 |
|
1523 // Test if object is forwarded. This is the case if lowest two bits are set. |
|
1524 Node* masked = new AndXNode(markword, phase->igvn().MakeConX(markOopDesc::lock_mask_in_place)); |
|
1525 phase->register_new_node(masked, ctrl); |
|
1526 Node* cmp = new CmpXNode(masked, phase->igvn().MakeConX(markOopDesc::marked_value)); |
|
1527 phase->register_new_node(cmp, ctrl); |
|
1528 |
|
1529 // Only branch to LRB stub if object is not forwarded; otherwise reply with fwd ptr |
|
1530 Node* bol = new BoolNode(cmp, BoolTest::eq); // Equals 3 means it's forwarded |
|
1531 phase->register_new_node(bol, ctrl); |
|
1532 |
|
1533 IfNode* iff = new IfNode(ctrl, bol, PROB_LIKELY(0.999), COUNT_UNKNOWN); |
|
1534 phase->register_control(iff, loop, ctrl); |
|
1535 Node* if_fwd = new IfTrueNode(iff); |
|
1536 phase->register_control(if_fwd, loop, iff); |
|
1537 Node* if_not_fwd = new IfFalseNode(iff); |
|
1538 phase->register_control(if_not_fwd, loop, iff); |
|
1539 |
|
1540 // Decode forward pointer: since we already have the lowest bits, we can just subtract them |
|
1541 // from the mark word without the need for large immediate mask. |
|
1542 Node* masked2 = new SubXNode(markword, masked); |
|
1543 phase->register_new_node(masked2, if_fwd); |
|
1544 Node* fwdraw = new CastX2PNode(masked2); |
|
1545 fwdraw->init_req(0, if_fwd); |
|
1546 phase->register_new_node(fwdraw, if_fwd); |
|
1547 Node* fwd = new CheckCastPPNode(NULL, fwdraw, val->bottom_type()); |
|
1548 phase->register_new_node(fwd, if_fwd); |
|
1549 |
|
1550 // Wire up not-equal-path in slots 3. |
|
1551 region->init_req(_fwded, if_fwd); |
|
1552 val_phi->init_req(_fwded, fwd); |
|
1553 raw_mem_phi->init_req(_fwded, raw_mem); |
|
1554 |
1492 |
1555 // Call lrb-stub and wire up that path in slots 4 |
1493 // Call lrb-stub and wire up that path in slots 4 |
1556 Node* result_mem = NULL; |
1494 Node* result_mem = NULL; |
1557 ctrl = if_not_fwd; |
1495 |
1558 fwd = new_val; |
1496 Node* fwd = new_val; |
1559 call_lrb_stub(ctrl, fwd, result_mem, raw_mem, phase); |
1497 Node* addr; |
|
1498 if (ShenandoahSelfFixing) { |
|
1499 VectorSet visited(Thread::current()->resource_area()); |
|
1500 addr = get_load_addr(phase, visited, lrb); |
|
1501 } else { |
|
1502 addr = phase->igvn().zerocon(T_OBJECT); |
|
1503 } |
|
1504 if (addr->Opcode() == Op_AddP) { |
|
1505 Node* orig_base = addr->in(AddPNode::Base); |
|
1506 Node* base = new CheckCastPPNode(ctrl, orig_base, orig_base->bottom_type(), true); |
|
1507 phase->register_new_node(base, ctrl); |
|
1508 if (addr->in(AddPNode::Base) == addr->in((AddPNode::Address))) { |
|
1509 // Field access |
|
1510 addr = addr->clone(); |
|
1511 addr->set_req(AddPNode::Base, base); |
|
1512 addr->set_req(AddPNode::Address, base); |
|
1513 phase->register_new_node(addr, ctrl); |
|
1514 } else { |
|
1515 Node* addr2 = addr->in(AddPNode::Address); |
|
1516 if (addr2->Opcode() == Op_AddP && addr2->in(AddPNode::Base) == addr2->in(AddPNode::Address) && |
|
1517 addr2->in(AddPNode::Base) == orig_base) { |
|
1518 addr2 = addr2->clone(); |
|
1519 addr2->set_req(AddPNode::Base, base); |
|
1520 addr2->set_req(AddPNode::Address, base); |
|
1521 phase->register_new_node(addr2, ctrl); |
|
1522 addr = addr->clone(); |
|
1523 addr->set_req(AddPNode::Base, base); |
|
1524 addr->set_req(AddPNode::Address, addr2); |
|
1525 phase->register_new_node(addr, ctrl); |
|
1526 } |
|
1527 } |
|
1528 } |
|
1529 call_lrb_stub(ctrl, fwd, addr, result_mem, raw_mem, lrb->is_native(), phase); |
1560 region->init_req(_evac_path, ctrl); |
1530 region->init_req(_evac_path, ctrl); |
1561 val_phi->init_req(_evac_path, fwd); |
1531 val_phi->init_req(_evac_path, fwd); |
1562 raw_mem_phi->init_req(_evac_path, result_mem); |
1532 raw_mem_phi->init_req(_evac_path, result_mem); |
1563 |
1533 |
1564 phase->register_control(region, loop, heap_stable_iff); |
1534 phase->register_control(region, loop, heap_stable_iff); |
1754 fixer.fix_mem(init_ctrl, region, init_raw_mem, raw_mem_for_ctrl, phi, uses); |
1724 fixer.fix_mem(init_ctrl, region, init_raw_mem, raw_mem_for_ctrl, phi, uses); |
1755 |
1725 |
1756 phase->igvn().replace_node(barrier, pre_val); |
1726 phase->igvn().replace_node(barrier, pre_val); |
1757 } |
1727 } |
1758 assert(state->enqueue_barriers_count() == 0, "all enqueue barrier nodes should have been replaced"); |
1728 assert(state->enqueue_barriers_count() == 0, "all enqueue barrier nodes should have been replaced"); |
|
1729 |
|
1730 } |
|
1731 |
|
1732 Node* ShenandoahBarrierC2Support::get_load_addr(PhaseIdealLoop* phase, VectorSet& visited, Node* in) { |
|
1733 if (visited.test_set(in->_idx)) { |
|
1734 return NULL; |
|
1735 } |
|
1736 switch (in->Opcode()) { |
|
1737 case Op_Proj: |
|
1738 return get_load_addr(phase, visited, in->in(0)); |
|
1739 case Op_CastPP: |
|
1740 case Op_CheckCastPP: |
|
1741 case Op_DecodeN: |
|
1742 case Op_EncodeP: |
|
1743 return get_load_addr(phase, visited, in->in(1)); |
|
1744 case Op_LoadN: |
|
1745 case Op_LoadP: |
|
1746 return in->in(MemNode::Address); |
|
1747 case Op_CompareAndExchangeN: |
|
1748 case Op_CompareAndExchangeP: |
|
1749 case Op_GetAndSetN: |
|
1750 case Op_GetAndSetP: |
|
1751 case Op_ShenandoahCompareAndExchangeP: |
|
1752 case Op_ShenandoahCompareAndExchangeN: |
|
1753 // Those instructions would just have stored a different |
|
1754 // value into the field. No use to attempt to fix it at this point. |
|
1755 return phase->igvn().zerocon(T_OBJECT); |
|
1756 case Op_CMoveP: |
|
1757 case Op_CMoveN: { |
|
1758 Node* t = get_load_addr(phase, visited, in->in(CMoveNode::IfTrue)); |
|
1759 Node* f = get_load_addr(phase, visited, in->in(CMoveNode::IfFalse)); |
|
1760 // Handle unambiguous cases: single address reported on both branches. |
|
1761 if (t != NULL && f == NULL) return t; |
|
1762 if (t == NULL && f != NULL) return f; |
|
1763 if (t != NULL && t == f) return t; |
|
1764 // Ambiguity. |
|
1765 return phase->igvn().zerocon(T_OBJECT); |
|
1766 } |
|
1767 case Op_Phi: { |
|
1768 Node* addr = NULL; |
|
1769 for (uint i = 1; i < in->req(); i++) { |
|
1770 Node* addr1 = get_load_addr(phase, visited, in->in(i)); |
|
1771 if (addr == NULL) { |
|
1772 addr = addr1; |
|
1773 } |
|
1774 if (addr != addr1) { |
|
1775 return phase->igvn().zerocon(T_OBJECT); |
|
1776 } |
|
1777 } |
|
1778 return addr; |
|
1779 } |
|
1780 case Op_ShenandoahLoadReferenceBarrier: |
|
1781 return get_load_addr(phase, visited, in->in(ShenandoahLoadReferenceBarrierNode::ValueIn)); |
|
1782 case Op_ShenandoahEnqueueBarrier: |
|
1783 return get_load_addr(phase, visited, in->in(1)); |
|
1784 case Op_CallDynamicJava: |
|
1785 case Op_CallLeaf: |
|
1786 case Op_CallStaticJava: |
|
1787 case Op_ConN: |
|
1788 case Op_ConP: |
|
1789 case Op_Parm: |
|
1790 return phase->igvn().zerocon(T_OBJECT); |
|
1791 default: |
|
1792 #ifdef ASSERT |
|
1793 fatal("Unknown node in get_load_addr: %s", NodeClassNames[in->Opcode()]); |
|
1794 #endif |
|
1795 return phase->igvn().zerocon(T_OBJECT); |
|
1796 } |
1759 |
1797 |
1760 } |
1798 } |
1761 |
1799 |
1762 void ShenandoahBarrierC2Support::move_heap_stable_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) { |
1800 void ShenandoahBarrierC2Support::move_heap_stable_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) { |
1763 IdealLoopTree *loop = phase->get_loop(iff); |
1801 IdealLoopTree *loop = phase->get_loop(iff); |
2975 u->adr_type() == NULL) { |
3013 u->adr_type() == NULL) { |
2976 assert(u->adr_type() != NULL || |
3014 assert(u->adr_type() != NULL || |
2977 u->Opcode() == Op_Rethrow || |
3015 u->Opcode() == Op_Rethrow || |
2978 u->Opcode() == Op_Return || |
3016 u->Opcode() == Op_Return || |
2979 u->Opcode() == Op_SafePoint || |
3017 u->Opcode() == Op_SafePoint || |
|
3018 u->Opcode() == Op_StoreIConditional || |
2980 u->Opcode() == Op_StoreLConditional || |
3019 u->Opcode() == Op_StoreLConditional || |
2981 (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) || |
3020 (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) || |
2982 (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) || |
3021 (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) || |
2983 u->Opcode() == Op_CallLeaf, ""); |
3022 u->Opcode() == Op_CallLeaf, "%s", u->Name()); |
2984 if (ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) { |
3023 if (ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) { |
2985 if (mm == NULL) { |
3024 if (mm == NULL) { |
2986 mm = allocate_merge_mem(mem, rep_proj, rep_ctrl); |
3025 mm = allocate_merge_mem(mem, rep_proj, rep_ctrl); |
2987 } |
3026 } |
2988 _phase->igvn().replace_input_of(u, u->find_edge(mem), mm); |
3027 _phase->igvn().replace_input_of(u, u->find_edge(mem), mm); |
3123 |
3182 |
3124 ShenandoahLoadReferenceBarrierNode::Strength ShenandoahLoadReferenceBarrierNode::get_barrier_strength() { |
3183 ShenandoahLoadReferenceBarrierNode::Strength ShenandoahLoadReferenceBarrierNode::get_barrier_strength() { |
3125 Unique_Node_List visited; |
3184 Unique_Node_List visited; |
3126 Node_Stack stack(0); |
3185 Node_Stack stack(0); |
3127 stack.push(this, 0); |
3186 stack.push(this, 0); |
|
3187 |
|
3188 // Look for strongest strength: go over nodes looking for STRONG ones. |
|
3189 // Stop once we encountered STRONG. Otherwise, walk until we ran out of nodes, |
|
3190 // and then the overall strength is NONE. |
3128 Strength strength = NONE; |
3191 Strength strength = NONE; |
3129 while (strength != STRONG && stack.size() > 0) { |
3192 while (strength != STRONG && stack.size() > 0) { |
3130 Node* n = stack.node(); |
3193 Node* n = stack.node(); |
3131 if (visited.member(n)) { |
3194 if (visited.member(n)) { |
3132 stack.pop(); |
3195 stack.pop(); |
3133 continue; |
3196 continue; |
3134 } |
3197 } |
3135 visited.push(n); |
3198 visited.push(n); |
3136 bool visit_users = false; |
3199 bool visit_users = false; |
3137 switch (n->Opcode()) { |
3200 switch (n->Opcode()) { |
3138 case Op_StoreN: |
3201 case Op_CallStaticJava: |
3139 case Op_StoreP: { |
|
3140 strength = STRONG; |
|
3141 break; |
|
3142 } |
|
3143 case Op_CmpP: { |
|
3144 if (!n->in(1)->bottom_type()->higher_equal(TypePtr::NULL_PTR) && |
|
3145 !n->in(2)->bottom_type()->higher_equal(TypePtr::NULL_PTR)) { |
|
3146 strength = STRONG; |
|
3147 } |
|
3148 break; |
|
3149 } |
|
3150 case Op_CallStaticJava: { |
|
3151 strength = STRONG; |
|
3152 break; |
|
3153 } |
|
3154 case Op_CallDynamicJava: |
3202 case Op_CallDynamicJava: |
3155 case Op_CallLeaf: |
3203 case Op_CallLeaf: |
3156 case Op_CallLeafNoFP: |
3204 case Op_CallLeafNoFP: |
3157 case Op_CompareAndSwapL: |
3205 case Op_CompareAndSwapL: |
3158 case Op_CompareAndSwapI: |
3206 case Op_CompareAndSwapI: |
3224 int alias_idx = Compile::current()->get_alias_index(adr_type); |
3291 int alias_idx = Compile::current()->get_alias_index(adr_type); |
3225 Compile::AliasType* alias_type = Compile::current()->alias_type(alias_idx); |
3292 Compile::AliasType* alias_type = Compile::current()->alias_type(alias_idx); |
3226 ciField* field = alias_type->field(); |
3293 ciField* field = alias_type->field(); |
3227 bool is_static = field != NULL && field->is_static(); |
3294 bool is_static = field != NULL && field->is_static(); |
3228 bool is_final = field != NULL && field->is_final(); |
3295 bool is_final = field != NULL && field->is_final(); |
3229 bool is_stable = field != NULL && field->is_stable(); |
3296 |
3230 if (ShenandoahOptimizeStaticFinals && is_static && is_final) { |
3297 if (ShenandoahOptimizeStaticFinals && is_static && is_final) { |
3231 // Leave strength as is. |
3298 // Loading the constant does not require barriers: it should be handled |
3232 } else if (ShenandoahOptimizeInstanceFinals && !is_static && is_final) { |
3299 // as part of GC roots already. |
3233 // Leave strength as is. |
|
3234 } else if (ShenandoahOptimizeStableFinals && (is_stable || (adr_type->isa_aryptr() && adr_type->isa_aryptr()->is_stable()))) { |
|
3235 // Leave strength as is. |
|
3236 } else { |
3300 } else { |
3237 strength = WEAK; |
3301 strength = STRONG; |
3238 } |
3302 } |
3239 break; |
3303 break; |
3240 } |
3304 } |
3241 case Op_AryEq: { |
|
3242 Node* n1 = n->in(2); |
|
3243 Node* n2 = n->in(3); |
|
3244 if (!ShenandoahOptimizeStableFinals || |
|
3245 !n1->bottom_type()->isa_aryptr() || !n1->bottom_type()->isa_aryptr()->is_stable() || |
|
3246 !n2->bottom_type()->isa_aryptr() || !n2->bottom_type()->isa_aryptr()->is_stable()) { |
|
3247 strength = WEAK; |
|
3248 } |
|
3249 break; |
|
3250 } |
|
3251 case Op_StrEquals: |
|
3252 case Op_StrComp: |
|
3253 case Op_StrIndexOf: |
|
3254 case Op_StrIndexOfChar: |
|
3255 if (!ShenandoahOptimizeStableFinals) { |
|
3256 strength = WEAK; |
|
3257 } |
|
3258 break; |
|
3259 case Op_Conv2B: |
3305 case Op_Conv2B: |
3260 case Op_LoadRange: |
3306 case Op_LoadRange: |
3261 case Op_LoadKlass: |
3307 case Op_LoadKlass: |
3262 case Op_LoadNKlass: |
3308 case Op_LoadNKlass: |
3263 // NONE, i.e. leave current strength as is |
3309 // Do not require barriers |
3264 break; |
3310 break; |
3265 case Op_AddP: |
3311 case Op_AddP: |
3266 case Op_CheckCastPP: |
3312 case Op_CheckCastPP: |
3267 case Op_CastPP: |
3313 case Op_CastPP: |
3268 case Op_CMoveP: |
3314 case Op_CMoveP: |
3269 case Op_Phi: |
3315 case Op_Phi: |
3270 case Op_ShenandoahLoadReferenceBarrier: |
3316 case Op_ShenandoahLoadReferenceBarrier: |
|
3317 // Whether or not these need the barriers depends on their users |
3271 visit_users = true; |
3318 visit_users = true; |
3272 break; |
3319 break; |
3273 default: { |
3320 default: { |
3274 #ifdef ASSERT |
3321 #ifdef ASSERT |
3275 tty->print_cr("Unknown node in get_barrier_strength:"); |
3322 fatal("Unknown node in get_barrier_strength: %s", NodeClassNames[n->Opcode()]); |
3276 n->dump(1); |
|
3277 ShouldNotReachHere(); |
|
3278 #else |
3323 #else |
|
3324 // Default to strong: better to have excess barriers, rather than miss some. |
3279 strength = STRONG; |
3325 strength = STRONG; |
3280 #endif |
3326 #endif |
3281 } |
3327 } |
3282 } |
3328 } |
3283 #ifdef ASSERT |
3329 |
3284 /* |
|
3285 if (strength == STRONG) { |
|
3286 tty->print("strengthening node: "); |
|
3287 n->dump(); |
|
3288 } |
|
3289 */ |
|
3290 #endif |
|
3291 stack.pop(); |
3330 stack.pop(); |
3292 if (visit_users) { |
3331 if (visit_users) { |
3293 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { |
3332 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { |
3294 Node* user = n->fast_out(i); |
3333 Node* user = n->fast_out(i); |
3295 if (user != NULL) { |
3334 if (user != NULL) { |