src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
branchdatagramsocketimpl-branch
changeset 58678 9cf78a70fa4f
parent 55079 de371e2d1acc
child 58679 9c3209ff7550
equal deleted inserted replaced
58677:13588c901957 58678:9cf78a70fa4f
   177   while (true) {
   177   while (true) {
   178     if (in->bottom_type() == TypePtr::NULL_PTR) {
   178     if (in->bottom_type() == TypePtr::NULL_PTR) {
   179       if (trace) {tty->print_cr("NULL");}
   179       if (trace) {tty->print_cr("NULL");}
   180     } else if (!in->bottom_type()->make_ptr()->make_oopptr()) {
   180     } else if (!in->bottom_type()->make_ptr()->make_oopptr()) {
   181       if (trace) {tty->print_cr("Non oop");}
   181       if (trace) {tty->print_cr("Non oop");}
   182     } else if (t == ShenandoahLoad && ShenandoahOptimizeStableFinals &&
       
   183                in->bottom_type()->make_ptr()->isa_aryptr() &&
       
   184                in->bottom_type()->make_ptr()->is_aryptr()->is_stable()) {
       
   185       if (trace) {tty->print_cr("Stable array load");}
       
   186     } else {
   182     } else {
   187       if (in->is_ConstraintCast()) {
   183       if (in->is_ConstraintCast()) {
   188         in = in->in(1);
   184         in = in->in(1);
   189         continue;
   185         continue;
   190       } else if (in->is_AddP()) {
   186       } else if (in->is_AddP()) {
   321           if (trace) {tty->print_cr("Mark load");}
   317           if (trace) {tty->print_cr("Mark load");}
   322         } else if (adr_type->isa_instptr() &&
   318         } else if (adr_type->isa_instptr() &&
   323                    adr_type->is_instptr()->klass()->is_subtype_of(Compile::current()->env()->Reference_klass()) &&
   319                    adr_type->is_instptr()->klass()->is_subtype_of(Compile::current()->env()->Reference_klass()) &&
   324                    adr_type->is_instptr()->offset() == java_lang_ref_Reference::referent_offset) {
   320                    adr_type->is_instptr()->offset() == java_lang_ref_Reference::referent_offset) {
   325           if (trace) {tty->print_cr("Reference.get()");}
   321           if (trace) {tty->print_cr("Reference.get()");}
   326         } else {
   322         } else if (!verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahLoad, trace, barriers_used)) {
   327           bool verify = true;
   323           report_verify_failure("Shenandoah verification: Load should have barriers", n);
   328           if (adr_type->isa_instptr()) {
       
   329             const TypeInstPtr* tinst = adr_type->is_instptr();
       
   330             ciKlass* k = tinst->klass();
       
   331             assert(k->is_instance_klass(), "");
       
   332             ciInstanceKlass* ik = (ciInstanceKlass*)k;
       
   333             int offset = adr_type->offset();
       
   334 
       
   335             if ((ik->debug_final_field_at(offset) && ShenandoahOptimizeInstanceFinals) ||
       
   336                 (ik->debug_stable_field_at(offset) && ShenandoahOptimizeStableFinals)) {
       
   337               if (trace) {tty->print_cr("Final/stable");}
       
   338               verify = false;
       
   339             } else if (k == ciEnv::current()->Class_klass() &&
       
   340                        tinst->const_oop() != NULL &&
       
   341                        tinst->offset() >= (ik->size_helper() * wordSize)) {
       
   342               ciInstanceKlass* k = tinst->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
       
   343               ciField* field = k->get_field_by_offset(tinst->offset(), true);
       
   344               if ((ShenandoahOptimizeStaticFinals && field->is_final()) ||
       
   345                   (ShenandoahOptimizeStableFinals && field->is_stable())) {
       
   346                 verify = false;
       
   347               }
       
   348             }
       
   349           }
       
   350 
       
   351           if (verify && !verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahLoad, trace, barriers_used)) {
       
   352             report_verify_failure("Shenandoah verification: Load should have barriers", n);
       
   353           }
       
   354         }
   324         }
   355       }
   325       }
   356     } else if (n->is_Store()) {
   326     } else if (n->is_Store()) {
   357       const bool trace = false;
   327       const bool trace = false;
   358 
   328 
   668             visited.Reset();
   638             visited.Reset();
   669           }
   639           }
   670         }
   640         }
   671       }
   641       }
   672     }
   642     }
   673     for( uint i = 0; i < n->len(); ++i ) {
       
   674       Node *m = n->in(i);
       
   675       if (m == NULL) continue;
       
   676 
       
   677       // In most cases, inputs should be known to be non null. If it's
       
   678       // not the case, it could be a missing cast_not_null() in an
       
   679       // intrinsic or support might be needed in AddPNode::Ideal() to
       
   680       // avoid a NULL+offset input.
       
   681       if (!(n->is_Phi() ||
       
   682             (n->is_SafePoint() && (!n->is_CallRuntime() || !strcmp(n->as_Call()->_name, "shenandoah_wb_pre") || !strcmp(n->as_Call()->_name, "unsafe_arraycopy"))) ||
       
   683             n->Opcode() == Op_CmpP ||
       
   684             n->Opcode() == Op_CmpN ||
       
   685             (n->Opcode() == Op_StoreP && i == StoreNode::ValueIn) ||
       
   686             (n->Opcode() == Op_StoreN && i == StoreNode::ValueIn) ||
       
   687             n->is_ConstraintCast() ||
       
   688             n->Opcode() == Op_Return ||
       
   689             n->Opcode() == Op_Conv2B ||
       
   690             n->is_AddP() ||
       
   691             n->Opcode() == Op_CMoveP ||
       
   692             n->Opcode() == Op_CMoveN ||
       
   693             n->Opcode() == Op_Rethrow ||
       
   694             n->is_MemBar() ||
       
   695             n->is_Mem() ||
       
   696             n->Opcode() == Op_AryEq ||
       
   697             n->Opcode() == Op_SCMemProj ||
       
   698             n->Opcode() == Op_EncodeP ||
       
   699             n->Opcode() == Op_DecodeN ||
       
   700             n->Opcode() == Op_ShenandoahEnqueueBarrier ||
       
   701             n->Opcode() == Op_ShenandoahLoadReferenceBarrier)) {
       
   702         if (m->bottom_type()->make_oopptr() && m->bottom_type()->make_oopptr()->meet(TypePtr::NULL_PTR) == m->bottom_type()) {
       
   703           report_verify_failure("Shenandoah verification: null input", n, m);
       
   704         }
       
   705       }
       
   706 
       
   707       wq.push(m);
       
   708     }
       
   709   }
   643   }
   710 
   644 
   711   if (verify_no_useless_barrier) {
   645   if (verify_no_useless_barrier) {
   712     for (int i = 0; i < barriers.length(); i++) {
   646     for (int i = 0; i < barriers.length(); i++) {
   713       Node* n = barriers.at(i);
   647       Node* n = barriers.at(i);
  1080 
  1014 
  1081   ctrl = new IfFalseNode(in_cset_fast_test_iff);
  1015   ctrl = new IfFalseNode(in_cset_fast_test_iff);
  1082   phase->register_control(ctrl, loop, in_cset_fast_test_iff);
  1016   phase->register_control(ctrl, loop, in_cset_fast_test_iff);
  1083 }
  1017 }
  1084 
  1018 
  1085 void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node*& result_mem, Node* raw_mem, PhaseIdealLoop* phase) {
  1019 void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem, bool is_native, PhaseIdealLoop* phase) {
  1086   IdealLoopTree*loop = phase->get_loop(ctrl);
  1020   IdealLoopTree*loop = phase->get_loop(ctrl);
  1087   const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst();
  1021   const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst();
  1088 
  1022 
  1089   // The slow path stub consumes and produces raw memory in addition
  1023   // The slow path stub consumes and produces raw memory in addition
  1090   // to the existing memory edges
  1024   // to the existing memory edges
  1091   Node* base = find_bottom_mem(ctrl, phase);
  1025   Node* base = find_bottom_mem(ctrl, phase);
  1092   MergeMemNode* mm = MergeMemNode::make(base);
  1026   MergeMemNode* mm = MergeMemNode::make(base);
  1093   mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
  1027   mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
  1094   phase->register_new_node(mm, ctrl);
  1028   phase->register_new_node(mm, ctrl);
  1095 
  1029 
  1096   Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT), "shenandoah_load_reference_barrier", TypeRawPtr::BOTTOM);
  1030   address target = LP64_ONLY(UseCompressedOops) NOT_LP64(false) ?
       
  1031           CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup_narrow) :
       
  1032           CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup);
       
  1033 
       
  1034   address calladdr = is_native ? CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native)
       
  1035                                : target;
       
  1036   const char* name = is_native ? "oop_load_from_native_barrier" : "load_reference_barrier";
       
  1037   Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(), calladdr, name, TypeRawPtr::BOTTOM);
       
  1038 
  1097   call->init_req(TypeFunc::Control, ctrl);
  1039   call->init_req(TypeFunc::Control, ctrl);
  1098   call->init_req(TypeFunc::I_O, phase->C->top());
  1040   call->init_req(TypeFunc::I_O, phase->C->top());
  1099   call->init_req(TypeFunc::Memory, mm);
  1041   call->init_req(TypeFunc::Memory, mm);
  1100   call->init_req(TypeFunc::FramePtr, phase->C->top());
  1042   call->init_req(TypeFunc::FramePtr, phase->C->top());
  1101   call->init_req(TypeFunc::ReturnAdr, phase->C->top());
  1043   call->init_req(TypeFunc::ReturnAdr, phase->C->top());
  1102   call->init_req(TypeFunc::Parms, val);
  1044   call->init_req(TypeFunc::Parms, val);
       
  1045   call->init_req(TypeFunc::Parms+1, load_addr);
  1103   phase->register_control(call, loop, ctrl);
  1046   phase->register_control(call, loop, ctrl);
  1104   ctrl = new ProjNode(call, TypeFunc::Control);
  1047   ctrl = new ProjNode(call, TypeFunc::Control);
  1105   phase->register_control(ctrl, loop, call);
  1048   phase->register_control(ctrl, loop, call);
  1106   result_mem = new ProjNode(call, TypeFunc::Memory);
  1049   result_mem = new ProjNode(call, TypeFunc::Memory);
  1107   phase->register_new_node(result_mem, call);
  1050   phase->register_new_node(result_mem, call);
  1327         }
  1270         }
  1328       }
  1271       }
  1329     }
  1272     }
  1330     if ((ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) || ctrl->is_CallJava()) {
  1273     if ((ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) || ctrl->is_CallJava()) {
  1331       CallNode* call = ctrl->is_Proj() ? ctrl->in(0)->as_CallJava() : ctrl->as_CallJava();
  1274       CallNode* call = ctrl->is_Proj() ? ctrl->in(0)->as_CallJava() : ctrl->as_CallJava();
       
  1275       if (call->entry_point() == OptoRuntime::rethrow_stub()) {
       
  1276         // The rethrow call may have too many projections to be
       
  1277         // properly handled here. Given there's no reason for a
       
  1278         // barrier to depend on the call, move it above the call
       
  1279         stack.push(lrb, 0);
       
  1280         do {
       
  1281           Node* n = stack.node();
       
  1282           uint idx = stack.index();
       
  1283           if (idx < n->req()) {
       
  1284             Node* in = n->in(idx);
       
  1285             stack.set_index(idx+1);
       
  1286             if (in != NULL) {
       
  1287               if (phase->has_ctrl(in)) {
       
  1288                 if (phase->is_dominator(call, phase->get_ctrl(in))) {
       
  1289 #ifdef ASSERT
       
  1290                   for (uint i = 0; i < stack.size(); i++) {
       
  1291                     assert(stack.node_at(i) != in, "node shouldn't have been seen yet");
       
  1292                   }
       
  1293 #endif
       
  1294                   stack.push(in, 0);
       
  1295                 }
       
  1296               } else {
       
  1297                 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call");
       
  1298               }
       
  1299             }
       
  1300           } else {
       
  1301             phase->set_ctrl(n, call->in(0));
       
  1302             stack.pop();
       
  1303           }
       
  1304         } while(stack.size() > 0);
       
  1305         continue;
       
  1306       }
  1332       CallProjections projs;
  1307       CallProjections projs;
  1333       call->extract_projections(&projs, false, false);
  1308       call->extract_projections(&projs, false, false);
  1334 
  1309 
  1335       Node* lrb_clone = lrb->clone();
  1310       Node* lrb_clone = lrb->clone();
  1336       phase->register_new_node(lrb_clone, projs.catchall_catchproj);
  1311       phase->register_new_node(lrb_clone, projs.catchall_catchproj);
  1462     Node* null_ctrl = NULL;
  1437     Node* null_ctrl = NULL;
  1463 
  1438 
  1464     assert(val->bottom_type()->make_oopptr(), "need oop");
  1439     assert(val->bottom_type()->make_oopptr(), "need oop");
  1465     assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant");
  1440     assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant");
  1466 
  1441 
  1467     enum { _heap_stable = 1, _not_cset, _fwded, _evac_path, _null_path, PATH_LIMIT };
  1442     enum { _heap_stable = 1, _not_cset, _evac_path, _null_path, PATH_LIMIT };
  1468     Node* region = new RegionNode(PATH_LIMIT);
  1443     Node* region = new RegionNode(PATH_LIMIT);
  1469     Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr());
  1444     Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr());
  1470     Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
  1445     Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
  1471 
  1446 
  1472     // Stable path.
  1447     // Stable path.
  1512       fix_null_check(unc, unc_ctrl, ctrl->in(0)->as_If()->proj_out(0), uses, phase);
  1487       fix_null_check(unc, unc_ctrl, ctrl->in(0)->as_If()->proj_out(0), uses, phase);
  1513 
  1488 
  1514       IfNode* iff = unc_ctrl->in(0)->as_If();
  1489       IfNode* iff = unc_ctrl->in(0)->as_If();
  1515       phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1));
  1490       phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1));
  1516     }
  1491     }
  1517     Node* addr = new AddPNode(new_val, uncasted_val, phase->igvn().MakeConX(oopDesc::mark_offset_in_bytes()));
       
  1518     phase->register_new_node(addr, ctrl);
       
  1519     assert(new_val->bottom_type()->isa_oopptr(), "what else?");
       
  1520     Node* markword = new LoadXNode(ctrl, raw_mem, addr, TypeRawPtr::BOTTOM, TypeX_X, MemNode::unordered);
       
  1521     phase->register_new_node(markword, ctrl);
       
  1522 
       
  1523     // Test if object is forwarded. This is the case if lowest two bits are set.
       
  1524     Node* masked = new AndXNode(markword, phase->igvn().MakeConX(markOopDesc::lock_mask_in_place));
       
  1525     phase->register_new_node(masked, ctrl);
       
  1526     Node* cmp = new CmpXNode(masked, phase->igvn().MakeConX(markOopDesc::marked_value));
       
  1527     phase->register_new_node(cmp, ctrl);
       
  1528 
       
  1529     // Only branch to LRB stub if object is not forwarded; otherwise reply with fwd ptr
       
  1530     Node* bol = new BoolNode(cmp, BoolTest::eq); // Equals 3 means it's forwarded
       
  1531     phase->register_new_node(bol, ctrl);
       
  1532 
       
  1533     IfNode* iff = new IfNode(ctrl, bol, PROB_LIKELY(0.999), COUNT_UNKNOWN);
       
  1534     phase->register_control(iff, loop, ctrl);
       
  1535     Node* if_fwd = new IfTrueNode(iff);
       
  1536     phase->register_control(if_fwd, loop, iff);
       
  1537     Node* if_not_fwd = new IfFalseNode(iff);
       
  1538     phase->register_control(if_not_fwd, loop, iff);
       
  1539 
       
  1540     // Decode forward pointer: since we already have the lowest bits, we can just subtract them
       
  1541     // from the mark word without the need for large immediate mask.
       
  1542     Node* masked2 = new SubXNode(markword, masked);
       
  1543     phase->register_new_node(masked2, if_fwd);
       
  1544     Node* fwdraw = new CastX2PNode(masked2);
       
  1545     fwdraw->init_req(0, if_fwd);
       
  1546     phase->register_new_node(fwdraw, if_fwd);
       
  1547     Node* fwd = new CheckCastPPNode(NULL, fwdraw, val->bottom_type());
       
  1548     phase->register_new_node(fwd, if_fwd);
       
  1549 
       
  1550     // Wire up not-equal-path in slots 3.
       
  1551     region->init_req(_fwded, if_fwd);
       
  1552     val_phi->init_req(_fwded, fwd);
       
  1553     raw_mem_phi->init_req(_fwded, raw_mem);
       
  1554 
  1492 
  1555     // Call lrb-stub and wire up that path in slots 4
  1493     // Call lrb-stub and wire up that path in slots 4
  1556     Node* result_mem = NULL;
  1494     Node* result_mem = NULL;
  1557     ctrl = if_not_fwd;
  1495 
  1558     fwd = new_val;
  1496     Node* fwd = new_val;
  1559     call_lrb_stub(ctrl, fwd, result_mem, raw_mem, phase);
  1497     Node* addr;
       
  1498     if (ShenandoahSelfFixing) {
       
  1499       VectorSet visited(Thread::current()->resource_area());
       
  1500       addr = get_load_addr(phase, visited, lrb);
       
  1501     } else {
       
  1502       addr = phase->igvn().zerocon(T_OBJECT);
       
  1503     }
       
  1504     if (addr->Opcode() == Op_AddP) {
       
  1505       Node* orig_base = addr->in(AddPNode::Base);
       
  1506       Node* base = new CheckCastPPNode(ctrl, orig_base, orig_base->bottom_type(), true);
       
  1507       phase->register_new_node(base, ctrl);
       
  1508       if (addr->in(AddPNode::Base) == addr->in((AddPNode::Address))) {
       
  1509         // Field access
       
  1510         addr = addr->clone();
       
  1511         addr->set_req(AddPNode::Base, base);
       
  1512         addr->set_req(AddPNode::Address, base);
       
  1513         phase->register_new_node(addr, ctrl);
       
  1514       } else {
       
  1515         Node* addr2 = addr->in(AddPNode::Address);
       
  1516         if (addr2->Opcode() == Op_AddP && addr2->in(AddPNode::Base) == addr2->in(AddPNode::Address) &&
       
  1517               addr2->in(AddPNode::Base) == orig_base) {
       
  1518           addr2 = addr2->clone();
       
  1519           addr2->set_req(AddPNode::Base, base);
       
  1520           addr2->set_req(AddPNode::Address, base);
       
  1521           phase->register_new_node(addr2, ctrl);
       
  1522           addr = addr->clone();
       
  1523           addr->set_req(AddPNode::Base, base);
       
  1524           addr->set_req(AddPNode::Address, addr2);
       
  1525           phase->register_new_node(addr, ctrl);
       
  1526         }
       
  1527       }
       
  1528     }
       
  1529     call_lrb_stub(ctrl, fwd, addr, result_mem, raw_mem, lrb->is_native(), phase);
  1560     region->init_req(_evac_path, ctrl);
  1530     region->init_req(_evac_path, ctrl);
  1561     val_phi->init_req(_evac_path, fwd);
  1531     val_phi->init_req(_evac_path, fwd);
  1562     raw_mem_phi->init_req(_evac_path, result_mem);
  1532     raw_mem_phi->init_req(_evac_path, result_mem);
  1563 
  1533 
  1564     phase->register_control(region, loop, heap_stable_iff);
  1534     phase->register_control(region, loop, heap_stable_iff);
  1754     fixer.fix_mem(init_ctrl, region, init_raw_mem, raw_mem_for_ctrl, phi, uses);
  1724     fixer.fix_mem(init_ctrl, region, init_raw_mem, raw_mem_for_ctrl, phi, uses);
  1755 
  1725 
  1756     phase->igvn().replace_node(barrier, pre_val);
  1726     phase->igvn().replace_node(barrier, pre_val);
  1757   }
  1727   }
  1758   assert(state->enqueue_barriers_count() == 0, "all enqueue barrier nodes should have been replaced");
  1728   assert(state->enqueue_barriers_count() == 0, "all enqueue barrier nodes should have been replaced");
       
  1729 
       
  1730 }
       
  1731 
       
  1732 Node* ShenandoahBarrierC2Support::get_load_addr(PhaseIdealLoop* phase, VectorSet& visited, Node* in) {
       
  1733   if (visited.test_set(in->_idx)) {
       
  1734     return NULL;
       
  1735   }
       
  1736   switch (in->Opcode()) {
       
  1737     case Op_Proj:
       
  1738       return get_load_addr(phase, visited, in->in(0));
       
  1739     case Op_CastPP:
       
  1740     case Op_CheckCastPP:
       
  1741     case Op_DecodeN:
       
  1742     case Op_EncodeP:
       
  1743       return get_load_addr(phase, visited, in->in(1));
       
  1744     case Op_LoadN:
       
  1745     case Op_LoadP:
       
  1746       return in->in(MemNode::Address);
       
  1747     case Op_CompareAndExchangeN:
       
  1748     case Op_CompareAndExchangeP:
       
  1749     case Op_GetAndSetN:
       
  1750     case Op_GetAndSetP:
       
  1751     case Op_ShenandoahCompareAndExchangeP:
       
  1752     case Op_ShenandoahCompareAndExchangeN:
       
  1753       // Those instructions would just have stored a different
       
  1754       // value into the field. No use to attempt to fix it at this point.
       
  1755       return phase->igvn().zerocon(T_OBJECT);
       
  1756     case Op_CMoveP:
       
  1757     case Op_CMoveN: {
       
  1758       Node* t = get_load_addr(phase, visited, in->in(CMoveNode::IfTrue));
       
  1759       Node* f = get_load_addr(phase, visited, in->in(CMoveNode::IfFalse));
       
  1760       // Handle unambiguous cases: single address reported on both branches.
       
  1761       if (t != NULL && f == NULL) return t;
       
  1762       if (t == NULL && f != NULL) return f;
       
  1763       if (t != NULL && t == f)    return t;
       
  1764       // Ambiguity.
       
  1765       return phase->igvn().zerocon(T_OBJECT);
       
  1766     }
       
  1767     case Op_Phi: {
       
  1768       Node* addr = NULL;
       
  1769       for (uint i = 1; i < in->req(); i++) {
       
  1770         Node* addr1 = get_load_addr(phase, visited, in->in(i));
       
  1771         if (addr == NULL) {
       
  1772           addr = addr1;
       
  1773         }
       
  1774         if (addr != addr1) {
       
  1775           return phase->igvn().zerocon(T_OBJECT);
       
  1776         }
       
  1777       }
       
  1778       return addr;
       
  1779     }
       
  1780     case Op_ShenandoahLoadReferenceBarrier:
       
  1781       return get_load_addr(phase, visited, in->in(ShenandoahLoadReferenceBarrierNode::ValueIn));
       
  1782     case Op_ShenandoahEnqueueBarrier:
       
  1783       return get_load_addr(phase, visited, in->in(1));
       
  1784     case Op_CallDynamicJava:
       
  1785     case Op_CallLeaf:
       
  1786     case Op_CallStaticJava:
       
  1787     case Op_ConN:
       
  1788     case Op_ConP:
       
  1789     case Op_Parm:
       
  1790       return phase->igvn().zerocon(T_OBJECT);
       
  1791     default:
       
  1792 #ifdef ASSERT
       
  1793       fatal("Unknown node in get_load_addr: %s", NodeClassNames[in->Opcode()]);
       
  1794 #endif
       
  1795       return phase->igvn().zerocon(T_OBJECT);
       
  1796   }
  1759 
  1797 
  1760 }
  1798 }
  1761 
  1799 
  1762 void ShenandoahBarrierC2Support::move_heap_stable_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) {
  1800 void ShenandoahBarrierC2Support::move_heap_stable_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) {
  1763   IdealLoopTree *loop = phase->get_loop(iff);
  1801   IdealLoopTree *loop = phase->get_loop(iff);
  2975                  u->adr_type() == NULL) {
  3013                  u->adr_type() == NULL) {
  2976         assert(u->adr_type() != NULL ||
  3014         assert(u->adr_type() != NULL ||
  2977                u->Opcode() == Op_Rethrow ||
  3015                u->Opcode() == Op_Rethrow ||
  2978                u->Opcode() == Op_Return ||
  3016                u->Opcode() == Op_Return ||
  2979                u->Opcode() == Op_SafePoint ||
  3017                u->Opcode() == Op_SafePoint ||
       
  3018                u->Opcode() == Op_StoreIConditional ||
  2980                u->Opcode() == Op_StoreLConditional ||
  3019                u->Opcode() == Op_StoreLConditional ||
  2981                (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) ||
  3020                (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) ||
  2982                (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) ||
  3021                (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) ||
  2983                u->Opcode() == Op_CallLeaf, "");
  3022                u->Opcode() == Op_CallLeaf, "%s", u->Name());
  2984         if (ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) {
  3023         if (ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) {
  2985           if (mm == NULL) {
  3024           if (mm == NULL) {
  2986             mm = allocate_merge_mem(mem, rep_proj, rep_ctrl);
  3025             mm = allocate_merge_mem(mem, rep_proj, rep_ctrl);
  2987           }
  3026           }
  2988           _phase->igvn().replace_input_of(u, u->find_edge(mem), mm);
  3027           _phase->igvn().replace_input_of(u, u->find_edge(mem), mm);
  2996       }
  3035       }
  2997     }
  3036     }
  2998   }
  3037   }
  2999 }
  3038 }
  3000 
  3039 
  3001 ShenandoahLoadReferenceBarrierNode::ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* obj)
  3040 ShenandoahLoadReferenceBarrierNode::ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* obj, bool native)
  3002 : Node(ctrl, obj) {
  3041 : Node(ctrl, obj), _native(native) {
  3003   ShenandoahBarrierSetC2::bsc2()->state()->add_load_reference_barrier(this);
  3042   ShenandoahBarrierSetC2::bsc2()->state()->add_load_reference_barrier(this);
       
  3043 }
       
  3044 
       
  3045 bool ShenandoahLoadReferenceBarrierNode::is_native() const {
       
  3046   return _native;
       
  3047 }
       
  3048 
       
  3049 uint ShenandoahLoadReferenceBarrierNode::size_of() const {
       
  3050   return sizeof(*this);
       
  3051 }
       
  3052 
       
  3053 uint ShenandoahLoadReferenceBarrierNode::hash() const {
       
  3054   return Node::hash() + (_native ? 1 : 0);
       
  3055 }
       
  3056 
       
  3057 bool ShenandoahLoadReferenceBarrierNode::cmp( const Node &n ) const {
       
  3058   return Node::cmp(n) && n.Opcode() == Op_ShenandoahLoadReferenceBarrier &&
       
  3059          _native == ((const ShenandoahLoadReferenceBarrierNode&)n)._native;
  3004 }
  3060 }
  3005 
  3061 
  3006 const Type* ShenandoahLoadReferenceBarrierNode::bottom_type() const {
  3062 const Type* ShenandoahLoadReferenceBarrierNode::bottom_type() const {
  3007   if (in(ValueIn) == NULL || in(ValueIn)->is_top()) {
  3063   if (in(ValueIn) == NULL || in(ValueIn)->is_top()) {
  3008     return Type::TOP;
  3064     return Type::TOP;
  3100     case Op_DecodeN:
  3156     case Op_DecodeN:
  3101     case Op_EncodeP:
  3157     case Op_EncodeP:
  3102       return needs_barrier_impl(phase, n->in(1), visited);
  3158       return needs_barrier_impl(phase, n->in(1), visited);
  3103     case Op_LoadN:
  3159     case Op_LoadN:
  3104       return true;
  3160       return true;
       
  3161     case Op_CMoveN:
  3105     case Op_CMoveP:
  3162     case Op_CMoveP:
  3106       return needs_barrier_impl(phase, n->in(2), visited) ||
  3163       return needs_barrier_impl(phase, n->in(2), visited) ||
  3107              needs_barrier_impl(phase, n->in(3), visited);
  3164              needs_barrier_impl(phase, n->in(3), visited);
  3108     case Op_ShenandoahEnqueueBarrier:
  3165     case Op_ShenandoahEnqueueBarrier:
  3109       return needs_barrier_impl(phase, n->in(1), visited);
  3166       return needs_barrier_impl(phase, n->in(1), visited);
       
  3167     case Op_CreateEx:
       
  3168       return false;
  3110     default:
  3169     default:
  3111       break;
  3170       break;
  3112   }
  3171   }
  3113 #ifdef ASSERT
  3172 #ifdef ASSERT
  3114   tty->print("need barrier on?: ");
  3173   tty->print("need barrier on?: ");
  3123 
  3182 
  3124 ShenandoahLoadReferenceBarrierNode::Strength ShenandoahLoadReferenceBarrierNode::get_barrier_strength() {
  3183 ShenandoahLoadReferenceBarrierNode::Strength ShenandoahLoadReferenceBarrierNode::get_barrier_strength() {
  3125   Unique_Node_List visited;
  3184   Unique_Node_List visited;
  3126   Node_Stack stack(0);
  3185   Node_Stack stack(0);
  3127   stack.push(this, 0);
  3186   stack.push(this, 0);
       
  3187 
       
  3188   // Look for strongest strength: go over nodes looking for STRONG ones.
       
  3189   // Stop once we encountered STRONG. Otherwise, walk until we ran out of nodes,
       
  3190   // and then the overall strength is NONE.
  3128   Strength strength = NONE;
  3191   Strength strength = NONE;
  3129   while (strength != STRONG && stack.size() > 0) {
  3192   while (strength != STRONG && stack.size() > 0) {
  3130     Node* n = stack.node();
  3193     Node* n = stack.node();
  3131     if (visited.member(n)) {
  3194     if (visited.member(n)) {
  3132       stack.pop();
  3195       stack.pop();
  3133       continue;
  3196       continue;
  3134     }
  3197     }
  3135     visited.push(n);
  3198     visited.push(n);
  3136     bool visit_users = false;
  3199     bool visit_users = false;
  3137     switch (n->Opcode()) {
  3200     switch (n->Opcode()) {
  3138       case Op_StoreN:
  3201       case Op_CallStaticJava:
  3139       case Op_StoreP: {
       
  3140         strength = STRONG;
       
  3141         break;
       
  3142       }
       
  3143       case Op_CmpP: {
       
  3144         if (!n->in(1)->bottom_type()->higher_equal(TypePtr::NULL_PTR) &&
       
  3145             !n->in(2)->bottom_type()->higher_equal(TypePtr::NULL_PTR)) {
       
  3146           strength = STRONG;
       
  3147         }
       
  3148         break;
       
  3149       }
       
  3150       case Op_CallStaticJava: {
       
  3151         strength = STRONG;
       
  3152         break;
       
  3153       }
       
  3154       case Op_CallDynamicJava:
  3202       case Op_CallDynamicJava:
  3155       case Op_CallLeaf:
  3203       case Op_CallLeaf:
  3156       case Op_CallLeafNoFP:
  3204       case Op_CallLeafNoFP:
  3157       case Op_CompareAndSwapL:
  3205       case Op_CompareAndSwapL:
  3158       case Op_CompareAndSwapI:
  3206       case Op_CompareAndSwapI:
  3198       case Op_StoreD:
  3246       case Op_StoreD:
  3199       case Op_StoreF:
  3247       case Op_StoreF:
  3200       case Op_StoreL:
  3248       case Op_StoreL:
  3201       case Op_StoreLConditional:
  3249       case Op_StoreLConditional:
  3202       case Op_StoreI:
  3250       case Op_StoreI:
       
  3251       case Op_StoreIConditional:
       
  3252       case Op_StoreN:
       
  3253       case Op_StoreP:
  3203       case Op_StoreVector:
  3254       case Op_StoreVector:
  3204       case Op_StrInflatedCopy:
  3255       case Op_StrInflatedCopy:
  3205       case Op_StrCompressedCopy:
  3256       case Op_StrCompressedCopy:
  3206       case Op_EncodeP:
  3257       case Op_EncodeP:
  3207       case Op_CastP2X:
  3258       case Op_CastP2X:
  3208       case Op_SafePoint:
  3259       case Op_SafePoint:
  3209       case Op_EncodeISOArray:
  3260       case Op_EncodeISOArray:
       
  3261       case Op_AryEq:
       
  3262       case Op_StrEquals:
       
  3263       case Op_StrComp:
       
  3264       case Op_StrIndexOf:
       
  3265       case Op_StrIndexOfChar:
       
  3266       case Op_HasNegatives:
       
  3267         // Known to require barriers
  3210         strength = STRONG;
  3268         strength = STRONG;
  3211         break;
  3269         break;
       
  3270       case Op_CmpP: {
       
  3271         if (n->in(1)->bottom_type()->higher_equal(TypePtr::NULL_PTR) ||
       
  3272             n->in(2)->bottom_type()->higher_equal(TypePtr::NULL_PTR)) {
       
  3273           // One of the sides is known null, no need for barrier.
       
  3274         } else {
       
  3275           strength = STRONG;
       
  3276         }
       
  3277         break;
       
  3278       }
  3212       case Op_LoadB:
  3279       case Op_LoadB:
  3213       case Op_LoadUB:
  3280       case Op_LoadUB:
  3214       case Op_LoadUS:
  3281       case Op_LoadUS:
  3215       case Op_LoadD:
  3282       case Op_LoadD:
  3216       case Op_LoadF:
  3283       case Op_LoadF:
  3224         int alias_idx = Compile::current()->get_alias_index(adr_type);
  3291         int alias_idx = Compile::current()->get_alias_index(adr_type);
  3225         Compile::AliasType* alias_type = Compile::current()->alias_type(alias_idx);
  3292         Compile::AliasType* alias_type = Compile::current()->alias_type(alias_idx);
  3226         ciField* field = alias_type->field();
  3293         ciField* field = alias_type->field();
  3227         bool is_static = field != NULL && field->is_static();
  3294         bool is_static = field != NULL && field->is_static();
  3228         bool is_final = field != NULL && field->is_final();
  3295         bool is_final = field != NULL && field->is_final();
  3229         bool is_stable = field != NULL && field->is_stable();
  3296 
  3230         if (ShenandoahOptimizeStaticFinals && is_static && is_final) {
  3297         if (ShenandoahOptimizeStaticFinals && is_static && is_final) {
  3231           // Leave strength as is.
  3298           // Loading the constant does not require barriers: it should be handled
  3232         } else if (ShenandoahOptimizeInstanceFinals && !is_static && is_final) {
  3299           // as part of GC roots already.
  3233           // Leave strength as is.
       
  3234         } else if (ShenandoahOptimizeStableFinals && (is_stable || (adr_type->isa_aryptr() && adr_type->isa_aryptr()->is_stable()))) {
       
  3235           // Leave strength as is.
       
  3236         } else {
  3300         } else {
  3237           strength = WEAK;
  3301           strength = STRONG;
  3238         }
  3302         }
  3239         break;
  3303         break;
  3240       }
  3304       }
  3241       case Op_AryEq: {
       
  3242         Node* n1 = n->in(2);
       
  3243         Node* n2 = n->in(3);
       
  3244         if (!ShenandoahOptimizeStableFinals ||
       
  3245             !n1->bottom_type()->isa_aryptr() || !n1->bottom_type()->isa_aryptr()->is_stable() ||
       
  3246             !n2->bottom_type()->isa_aryptr() || !n2->bottom_type()->isa_aryptr()->is_stable()) {
       
  3247           strength = WEAK;
       
  3248         }
       
  3249         break;
       
  3250       }
       
  3251       case Op_StrEquals:
       
  3252       case Op_StrComp:
       
  3253       case Op_StrIndexOf:
       
  3254       case Op_StrIndexOfChar:
       
  3255         if (!ShenandoahOptimizeStableFinals) {
       
  3256            strength = WEAK;
       
  3257         }
       
  3258         break;
       
  3259       case Op_Conv2B:
  3305       case Op_Conv2B:
  3260       case Op_LoadRange:
  3306       case Op_LoadRange:
  3261       case Op_LoadKlass:
  3307       case Op_LoadKlass:
  3262       case Op_LoadNKlass:
  3308       case Op_LoadNKlass:
  3263         // NONE, i.e. leave current strength as is
  3309         // Do not require barriers
  3264         break;
  3310         break;
  3265       case Op_AddP:
  3311       case Op_AddP:
  3266       case Op_CheckCastPP:
  3312       case Op_CheckCastPP:
  3267       case Op_CastPP:
  3313       case Op_CastPP:
  3268       case Op_CMoveP:
  3314       case Op_CMoveP:
  3269       case Op_Phi:
  3315       case Op_Phi:
  3270       case Op_ShenandoahLoadReferenceBarrier:
  3316       case Op_ShenandoahLoadReferenceBarrier:
       
  3317         // Whether or not these need the barriers depends on their users
  3271         visit_users = true;
  3318         visit_users = true;
  3272         break;
  3319         break;
  3273       default: {
  3320       default: {
  3274 #ifdef ASSERT
  3321 #ifdef ASSERT
  3275         tty->print_cr("Unknown node in get_barrier_strength:");
  3322         fatal("Unknown node in get_barrier_strength: %s", NodeClassNames[n->Opcode()]);
  3276         n->dump(1);
       
  3277         ShouldNotReachHere();
       
  3278 #else
  3323 #else
       
  3324         // Default to strong: better to have excess barriers, rather than miss some.
  3279         strength = STRONG;
  3325         strength = STRONG;
  3280 #endif
  3326 #endif
  3281       }
  3327       }
  3282     }
  3328     }
  3283 #ifdef ASSERT
  3329 
  3284 /*
       
  3285     if (strength == STRONG) {
       
  3286       tty->print("strengthening node: ");
       
  3287       n->dump();
       
  3288     }
       
  3289     */
       
  3290 #endif
       
  3291     stack.pop();
  3330     stack.pop();
  3292     if (visit_users) {
  3331     if (visit_users) {
  3293       for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
  3332       for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
  3294         Node* user = n->fast_out(i);
  3333         Node* user = n->fast_out(i);
  3295         if (user != NULL) {
  3334         if (user != NULL) {