Mercurial Mercurial > jdk-sandbox / comparison
summary | shortlog | changelog | graph | tags | bookmarks | branches | files | changeset | file | latest | revisions | annotate | diff | comparison | raw | help
Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
changeset 58219 bc0648405d67
parent 57777 90ead0febf56
child 58366 222a91b9438a
equal deleted inserted replaced
58218:0d7877278adf 58219:bc0648405d67 
  1014 
  1014 
  1015   ctrl = new IfFalseNode(in_cset_fast_test_iff);
 
  1015   ctrl = new IfFalseNode(in_cset_fast_test_iff);
 
  1016   phase->register_control(ctrl, loop, in_cset_fast_test_iff);
 
  1016   phase->register_control(ctrl, loop, in_cset_fast_test_iff);
 
  1017 }
 
  1017 }
 
  1018 
  1018 
  1019 void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node*& result_mem, Node* raw_mem, bool is_native, PhaseIdealLoop* phase) {
 
  1019 void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem, bool is_native, PhaseIdealLoop* phase) {
 
  1020   IdealLoopTree*loop = phase->get_loop(ctrl);
 
  1020   IdealLoopTree*loop = phase->get_loop(ctrl);
 
  1021   const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst();
 
  1021   const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst();
 
  1022 
  1022 
  1023   // The slow path stub consumes and produces raw memory in addition
 
  1023   // The slow path stub consumes and produces raw memory in addition
 
  1024   // to the existing memory edges
 
  1024   // to the existing memory edges
 
  1025   Node* base = find_bottom_mem(ctrl, phase);
 
  1025   Node* base = find_bottom_mem(ctrl, phase);
 
  1026   MergeMemNode* mm = MergeMemNode::make(base);
 
  1026   MergeMemNode* mm = MergeMemNode::make(base);
 
  1027   mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
 
  1027   mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
 
  1028   phase->register_new_node(mm, ctrl);
 
  1028   phase->register_new_node(mm, ctrl);
 
  1029 
  1029 
       
  1030   address target = LP64_ONLY(UseCompressedOops) NOT_LP64(false) ?
 
       
  1031           CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup_narrow) :
 
       
  1032           CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup);
 
       
  1033 
  1030   address calladdr = is_native ? CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native)
 
  1034   address calladdr = is_native ? CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native)
 
  1031                                : CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier);
 
  1035                                : target;
 
  1032   const char* name = is_native ? "oop_load_from_native_barrier" : "load_reference_barrier";
 
  1036   const char* name = is_native ? "oop_load_from_native_barrier" : "load_reference_barrier";
 
  1033   Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(), calladdr, name, TypeRawPtr::BOTTOM);
 
  1037   Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(), calladdr, name, TypeRawPtr::BOTTOM);
 
       
  1038 
  1034   call->init_req(TypeFunc::Control, ctrl);
 
  1039   call->init_req(TypeFunc::Control, ctrl);
 
  1035   call->init_req(TypeFunc::I_O, phase->C->top());
 
  1040   call->init_req(TypeFunc::I_O, phase->C->top());
 
  1036   call->init_req(TypeFunc::Memory, mm);
 
  1041   call->init_req(TypeFunc::Memory, mm);
 
  1037   call->init_req(TypeFunc::FramePtr, phase->C->top());
 
  1042   call->init_req(TypeFunc::FramePtr, phase->C->top());
 
  1038   call->init_req(TypeFunc::ReturnAdr, phase->C->top());
 
  1043   call->init_req(TypeFunc::ReturnAdr, phase->C->top());
 
  1039   call->init_req(TypeFunc::Parms, val);
 
  1044   call->init_req(TypeFunc::Parms, val);
 
       
  1045   call->init_req(TypeFunc::Parms+1, load_addr);
 
  1040   phase->register_control(call, loop, ctrl);
 
  1046   phase->register_control(call, loop, ctrl);
 
  1041   ctrl = new ProjNode(call, TypeFunc::Control);
 
  1047   ctrl = new ProjNode(call, TypeFunc::Control);
 
  1042   phase->register_control(ctrl, loop, call);
 
  1048   phase->register_control(ctrl, loop, call);
 
  1043   result_mem = new ProjNode(call, TypeFunc::Memory);
 
  1049   result_mem = new ProjNode(call, TypeFunc::Memory);
 
  1044   phase->register_new_node(result_mem, call);
 
  1050   phase->register_new_node(result_mem, call);
 
  1399     Node* null_ctrl = NULL;
 
  1405     Node* null_ctrl = NULL;
 
  1400 
  1406 
  1401     assert(val->bottom_type()->make_oopptr(), "need oop");
 
  1407     assert(val->bottom_type()->make_oopptr(), "need oop");
 
  1402     assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant");
 
  1408     assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant");
 
  1403 
  1409 
  1404     enum { _heap_stable = 1, _not_cset, _fwded, _evac_path, _null_path, PATH_LIMIT };
 
  1410     enum { _heap_stable = 1, _not_cset, _evac_path, _null_path, PATH_LIMIT };
 
  1405     Node* region = new RegionNode(PATH_LIMIT);
 
  1411     Node* region = new RegionNode(PATH_LIMIT);
 
  1406     Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr());
 
  1412     Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr());
 
  1407     Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
 
  1413     Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
 
  1408 
  1414 
  1409     // Stable path.
 
  1415     // Stable path.
 
  1449       fix_null_check(unc, unc_ctrl, ctrl->in(0)->as_If()->proj_out(0), uses, phase);
 
  1455       fix_null_check(unc, unc_ctrl, ctrl->in(0)->as_If()->proj_out(0), uses, phase);
 
  1450 
  1456 
  1451       IfNode* iff = unc_ctrl->in(0)->as_If();
 
  1457       IfNode* iff = unc_ctrl->in(0)->as_If();
 
  1452       phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1));
 
  1458       phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1));
 
  1453     }
 
  1459     }
 
  1454     Node* addr = new AddPNode(new_val, uncasted_val, phase->igvn().MakeConX(oopDesc::mark_offset_in_bytes()));
 
       
  1455     phase->register_new_node(addr, ctrl);
 
       
  1456     assert(new_val->bottom_type()->isa_oopptr(), "what else?");
 
       
  1457     Node* markword = new LoadXNode(ctrl, raw_mem, addr, TypeRawPtr::BOTTOM, TypeX_X, MemNode::unordered);
 
       
  1458     phase->register_new_node(markword, ctrl);
 
       
  1459 
       
  1460     // Test if object is forwarded. This is the case if lowest two bits are set.
 
       
  1461     Node* masked = new AndXNode(markword, phase->igvn().MakeConX(markWord::lock_mask_in_place));
 
       
  1462     phase->register_new_node(masked, ctrl);
 
       
  1463     Node* cmp = new CmpXNode(masked, phase->igvn().MakeConX(markWord::marked_value));
 
       
  1464     phase->register_new_node(cmp, ctrl);
 
       
  1465 
       
  1466     // Only branch to LRB stub if object is not forwarded; otherwise reply with fwd ptr
 
       
  1467     Node* bol = new BoolNode(cmp, BoolTest::eq); // Equals 3 means it's forwarded
 
       
  1468     phase->register_new_node(bol, ctrl);
 
       
  1469 
       
  1470     IfNode* iff = new IfNode(ctrl, bol, PROB_LIKELY(0.999), COUNT_UNKNOWN);
 
       
  1471     phase->register_control(iff, loop, ctrl);
 
       
  1472     Node* if_fwd = new IfTrueNode(iff);
 
       
  1473     phase->register_control(if_fwd, loop, iff);
 
       
  1474     Node* if_not_fwd = new IfFalseNode(iff);
 
       
  1475     phase->register_control(if_not_fwd, loop, iff);
 
       
  1476 
       
  1477     // Decode forward pointer: since we already have the lowest bits, we can just subtract them
 
       
  1478     // from the mark word without the need for large immediate mask.
 
       
  1479     Node* masked2 = new SubXNode(markword, masked);
 
       
  1480     phase->register_new_node(masked2, if_fwd);
 
       
  1481     Node* fwdraw = new CastX2PNode(masked2);
 
       
  1482     fwdraw->init_req(0, if_fwd);
 
       
  1483     phase->register_new_node(fwdraw, if_fwd);
 
       
  1484     Node* fwd = new CheckCastPPNode(NULL, fwdraw, val->bottom_type());
 
       
  1485     phase->register_new_node(fwd, if_fwd);
 
       
  1486 
       
  1487     // Wire up not-equal-path in slots 3.
 
       
  1488     region->init_req(_fwded, if_fwd);
 
       
  1489     val_phi->init_req(_fwded, fwd);
 
       
  1490     raw_mem_phi->init_req(_fwded, raw_mem);
 
       
  1491 
  1460 
  1492     // Call lrb-stub and wire up that path in slots 4
 
  1461     // Call lrb-stub and wire up that path in slots 4
 
  1493     Node* result_mem = NULL;
 
  1462     Node* result_mem = NULL;
 
  1494     ctrl = if_not_fwd;
 
  1463 
  1495     fwd = new_val;
 
  1464     Node* fwd = new_val;
 
  1496     call_lrb_stub(ctrl, fwd, result_mem, raw_mem, lrb->is_native(), phase);
 
  1465     Node* addr;
 
       
  1466     if (ShenandoahSelfFixing) {
 
       
  1467       VectorSet visited(Thread::current()->resource_area());
 
       
  1468       addr = get_load_addr(phase, visited, lrb);
 
       
  1469     } else {
 
       
  1470       addr = phase->igvn().zerocon(T_OBJECT);
 
       
  1471     }
 
       
  1472     if (addr->Opcode() == Op_AddP) {
 
       
  1473       Node* orig_base = addr->in(AddPNode::Base);
 
       
  1474       Node* base = new CheckCastPPNode(ctrl, orig_base, orig_base->bottom_type(), true);
 
       
  1475       phase->register_new_node(base, ctrl);
 
       
  1476       if (addr->in(AddPNode::Base) == addr->in((AddPNode::Address))) {
 
       
  1477         // Field access
 
       
  1478         addr = addr->clone();
 
       
  1479         addr->set_req(AddPNode::Base, base);
 
       
  1480         addr->set_req(AddPNode::Address, base);
 
       
  1481         phase->register_new_node(addr, ctrl);
 
       
  1482       } else {
 
       
  1483         Node* addr2 = addr->in(AddPNode::Address);
 
       
  1484         if (addr2->Opcode() == Op_AddP && addr2->in(AddPNode::Base) == addr2->in(AddPNode::Address) &&
 
       
  1485               addr2->in(AddPNode::Base) == orig_base) {
 
       
  1486           addr2 = addr2->clone();
 
       
  1487           addr2->set_req(AddPNode::Base, base);
 
       
  1488           addr2->set_req(AddPNode::Address, base);
 
       
  1489           phase->register_new_node(addr2, ctrl);
 
       
  1490           addr = addr->clone();
 
       
  1491           addr->set_req(AddPNode::Base, base);
 
       
  1492           addr->set_req(AddPNode::Address, addr2);
 
       
  1493           phase->register_new_node(addr, ctrl);
 
       
  1494         }
 
       
  1495       }
 
       
  1496     }
 
       
  1497     call_lrb_stub(ctrl, fwd, addr, result_mem, raw_mem, lrb->is_native(), phase);
 
  1497     region->init_req(_evac_path, ctrl);
 
  1498     region->init_req(_evac_path, ctrl);
 
  1498     val_phi->init_req(_evac_path, fwd);
 
  1499     val_phi->init_req(_evac_path, fwd);
 
  1499     raw_mem_phi->init_req(_evac_path, result_mem);
 
  1500     raw_mem_phi->init_req(_evac_path, result_mem);
 
  1500 
  1501 
  1501     phase->register_control(region, loop, heap_stable_iff);
 
  1502     phase->register_control(region, loop, heap_stable_iff);
 
  1691     fixer.fix_mem(init_ctrl, region, init_raw_mem, raw_mem_for_ctrl, phi, uses);
 
  1692     fixer.fix_mem(init_ctrl, region, init_raw_mem, raw_mem_for_ctrl, phi, uses);
 
  1692 
  1693 
  1693     phase->igvn().replace_node(barrier, pre_val);
 
  1694     phase->igvn().replace_node(barrier, pre_val);
 
  1694   }
 
  1695   }
 
  1695   assert(state->enqueue_barriers_count() == 0, "all enqueue barrier nodes should have been replaced");
 
  1696   assert(state->enqueue_barriers_count() == 0, "all enqueue barrier nodes should have been replaced");
 
       
  1697 
       
  1698 }
 
       
  1699 
       
  1700 Node* ShenandoahBarrierC2Support::get_load_addr(PhaseIdealLoop* phase, VectorSet& visited, Node* in) {
 
       
  1701   if (visited.test_set(in->_idx)) {
 
       
  1702     return NULL;
 
       
  1703   }
 
       
  1704   switch (in->Opcode()) {
 
       
  1705     case Op_Proj:
 
       
  1706       return get_load_addr(phase, visited, in->in(0));
 
       
  1707     case Op_CastPP:
 
       
  1708     case Op_CheckCastPP:
 
       
  1709     case Op_DecodeN:
 
       
  1710     case Op_EncodeP:
 
       
  1711       return get_load_addr(phase, visited, in->in(1));
 
       
  1712     case Op_LoadN:
 
       
  1713     case Op_LoadP:
 
       
  1714       return in->in(MemNode::Address);
 
       
  1715     case Op_CompareAndExchangeN:
 
       
  1716     case Op_CompareAndExchangeP:
 
       
  1717     case Op_GetAndSetN:
 
       
  1718     case Op_GetAndSetP:
 
       
  1719     case Op_ShenandoahCompareAndExchangeP:
 
       
  1720     case Op_ShenandoahCompareAndExchangeN:
 
       
  1721       // Those instructions would just have stored a different
 
       
  1722       // value into the field. No use to attempt to fix it at this point.
 
       
  1723       return phase->igvn().zerocon(T_OBJECT);
 
       
  1724     case Op_CMoveP:
 
       
  1725     case Op_CMoveN: {
 
       
  1726       Node* t = get_load_addr(phase, visited, in->in(CMoveNode::IfTrue));
 
       
  1727       Node* f = get_load_addr(phase, visited, in->in(CMoveNode::IfFalse));
 
       
  1728       // Handle unambiguous cases: single address reported on both branches.
 
       
  1729       if (t != NULL && f == NULL) return t;
 
       
  1730       if (t == NULL && f != NULL) return f;
 
       
  1731       if (t != NULL && t == f)    return t;
 
       
  1732       // Ambiguity.
 
       
  1733       return phase->igvn().zerocon(T_OBJECT);
 
       
  1734     }
 
       
  1735     case Op_Phi: {
 
       
  1736       Node* addr = NULL;
 
       
  1737       for (uint i = 1; i < in->req(); i++) {
 
       
  1738         Node* addr1 = get_load_addr(phase, visited, in->in(i));
 
       
  1739         if (addr == NULL) {
 
       
  1740           addr = addr1;
 
       
  1741         }
 
       
  1742         if (addr != addr1) {
 
       
  1743           return phase->igvn().zerocon(T_OBJECT);
 
       
  1744         }
 
       
  1745       }
 
       
  1746       return addr;
 
       
  1747     }
 
       
  1748     case Op_ShenandoahLoadReferenceBarrier:
 
       
  1749       return get_load_addr(phase, visited, in->in(ShenandoahLoadReferenceBarrierNode::ValueIn));
 
       
  1750     case Op_ShenandoahEnqueueBarrier:
 
       
  1751       return get_load_addr(phase, visited, in->in(1));
 
       
  1752     case Op_CallDynamicJava:
 
       
  1753     case Op_CallLeaf:
 
       
  1754     case Op_CallStaticJava:
 
       
  1755     case Op_ConN:
 
       
  1756     case Op_ConP:
 
       
  1757     case Op_Parm:
 
       
  1758       return phase->igvn().zerocon(T_OBJECT);
 
       
  1759     default:
 
       
  1760 #ifdef ASSERT
 
       
  1761       fatal("Unknown node in get_load_addr: %s", NodeClassNames[in->Opcode()]);
 
       
  1762 #endif
 
       
  1763       return phase->igvn().zerocon(T_OBJECT);
 
       
  1764   }
 
  1696 
  1765 
  1697 }
 
  1766 }
 
  1698 
  1767 
  1699 void ShenandoahBarrierC2Support::move_heap_stable_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) {
 
  1768 void ShenandoahBarrierC2Support::move_heap_stable_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) {
 
  1700   IdealLoopTree *loop = phase->get_loop(iff);
 
  1769   IdealLoopTree *loop = phase->get_loop(iff);
jdk-sandbox