diff -r f562f8318ebd -r 6c0ab8bd8da5 src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp Thu Apr 04 07:43:44 2019 -0700 +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp Tue Apr 02 23:00:22 2019 +0200 @@ -43,121 +43,56 @@ } ShenandoahBarrierSetC2State::ShenandoahBarrierSetC2State(Arena* comp_arena) - : _shenandoah_barriers(new (comp_arena) GrowableArray(comp_arena, 8, 0, NULL)) { + : _enqueue_barriers(new (comp_arena) GrowableArray(comp_arena, 8, 0, NULL)), + _load_reference_barriers(new (comp_arena) GrowableArray(comp_arena, 8, 0, NULL)) { } -int ShenandoahBarrierSetC2State::shenandoah_barriers_count() const { - return _shenandoah_barriers->length(); +int ShenandoahBarrierSetC2State::enqueue_barriers_count() const { + return _enqueue_barriers->length(); } -ShenandoahWriteBarrierNode* ShenandoahBarrierSetC2State::shenandoah_barrier(int idx) const { - return _shenandoah_barriers->at(idx); +ShenandoahEnqueueBarrierNode* ShenandoahBarrierSetC2State::enqueue_barrier(int idx) const { + return _enqueue_barriers->at(idx); } -void ShenandoahBarrierSetC2State::add_shenandoah_barrier(ShenandoahWriteBarrierNode * n) { - assert(!_shenandoah_barriers->contains(n), "duplicate entry in barrier list"); - _shenandoah_barriers->append(n); +void ShenandoahBarrierSetC2State::add_enqueue_barrier(ShenandoahEnqueueBarrierNode * n) { + assert(!_enqueue_barriers->contains(n), "duplicate entry in barrier list"); + _enqueue_barriers->append(n); } -void ShenandoahBarrierSetC2State::remove_shenandoah_barrier(ShenandoahWriteBarrierNode * n) { - if (_shenandoah_barriers->contains(n)) { - _shenandoah_barriers->remove(n); +void ShenandoahBarrierSetC2State::remove_enqueue_barrier(ShenandoahEnqueueBarrierNode * n) { + if (_enqueue_barriers->contains(n)) { + _enqueue_barriers->remove(n); } } -#define __ kit-> +int ShenandoahBarrierSetC2State::load_reference_barriers_count() const { + return _load_reference_barriers->length(); +} + +ShenandoahLoadReferenceBarrierNode* ShenandoahBarrierSetC2State::load_reference_barrier(int idx) const { + return _load_reference_barriers->at(idx); +} -Node* ShenandoahBarrierSetC2::shenandoah_read_barrier(GraphKit* kit, Node* obj) const { - if (ShenandoahReadBarrier) { - obj = shenandoah_read_barrier_impl(kit, obj, false, true, true); +void ShenandoahBarrierSetC2State::add_load_reference_barrier(ShenandoahLoadReferenceBarrierNode * n) { + assert(!_load_reference_barriers->contains(n), "duplicate entry in barrier list"); + _load_reference_barriers->append(n); +} + +void ShenandoahBarrierSetC2State::remove_load_reference_barrier(ShenandoahLoadReferenceBarrierNode * n) { + if (_load_reference_barriers->contains(n)) { + _load_reference_barriers->remove(n); } - return obj; } Node* ShenandoahBarrierSetC2::shenandoah_storeval_barrier(GraphKit* kit, Node* obj) const { if (ShenandoahStoreValEnqueueBarrier) { - obj = shenandoah_write_barrier(kit, obj); obj = shenandoah_enqueue_barrier(kit, obj); } - if (ShenandoahStoreValReadBarrier) { - obj = shenandoah_read_barrier_impl(kit, obj, true, false, false); - } return obj; } -Node* ShenandoahBarrierSetC2::shenandoah_read_barrier_impl(GraphKit* kit, Node* obj, bool use_ctrl, bool use_mem, bool allow_fromspace) const { - const Type* obj_type = obj->bottom_type(); - if (obj_type->higher_equal(TypePtr::NULL_PTR)) { - return obj; - } - const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type); - Node* mem = use_mem ? __ memory(adr_type) : __ immutable_memory(); - - if (! ShenandoahBarrierNode::needs_barrier(&__ gvn(), NULL, obj, mem, allow_fromspace)) { - // We know it is null, no barrier needed. - return obj; - } - - if (obj_type->meet(TypePtr::NULL_PTR) == obj_type->remove_speculative()) { - - // We don't know if it's null or not. Need null-check. - enum { _not_null_path = 1, _null_path, PATH_LIMIT }; - RegionNode* region = new RegionNode(PATH_LIMIT); - Node* phi = new PhiNode(region, obj_type); - Node* null_ctrl = __ top(); - Node* not_null_obj = __ null_check_oop(obj, &null_ctrl); - - region->init_req(_null_path, null_ctrl); - phi ->init_req(_null_path, __ zerocon(T_OBJECT)); - - Node* ctrl = use_ctrl ? __ control() : NULL; - ShenandoahReadBarrierNode* rb = new ShenandoahReadBarrierNode(ctrl, mem, not_null_obj, allow_fromspace); - Node* n = __ gvn().transform(rb); - - region->init_req(_not_null_path, __ control()); - phi ->init_req(_not_null_path, n); - - __ set_control(__ gvn().transform(region)); - __ record_for_igvn(region); - return __ gvn().transform(phi); - - } else { - // We know it is not null. Simple barrier is sufficient. - Node* ctrl = use_ctrl ? __ control() : NULL; - ShenandoahReadBarrierNode* rb = new ShenandoahReadBarrierNode(ctrl, mem, obj, allow_fromspace); - Node* n = __ gvn().transform(rb); - __ record_for_igvn(n); - return n; - } -} - -Node* ShenandoahBarrierSetC2::shenandoah_write_barrier_helper(GraphKit* kit, Node* obj, const TypePtr* adr_type) const { - ShenandoahWriteBarrierNode* wb = new ShenandoahWriteBarrierNode(kit->C, kit->control(), kit->memory(adr_type), obj); - Node* n = __ gvn().transform(wb); - if (n == wb) { // New barrier needs memory projection. - Node* proj = __ gvn().transform(new ShenandoahWBMemProjNode(n)); - __ set_memory(proj, adr_type); - } - return n; -} - -Node* ShenandoahBarrierSetC2::shenandoah_write_barrier(GraphKit* kit, Node* obj) const { - if (ShenandoahWriteBarrier) { - obj = shenandoah_write_barrier_impl(kit, obj); - } - return obj; -} - -Node* ShenandoahBarrierSetC2::shenandoah_write_barrier_impl(GraphKit* kit, Node* obj) const { - if (! ShenandoahBarrierNode::needs_barrier(&__ gvn(), NULL, obj, NULL, true)) { - return obj; - } - const Type* obj_type = obj->bottom_type(); - const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type); - Node* n = shenandoah_write_barrier_helper(kit, obj, adr_type); - __ record_for_igvn(n); - return n; -} +#define __ kit-> bool ShenandoahBarrierSetC2::satb_can_remove_pre_barrier(GraphKit* kit, PhaseTransform* phase, Node* adr, BasicType bt, uint adr_idx) const { @@ -304,7 +239,7 @@ Node* gc_state = __ AddP(no_base, tls, __ ConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()))); Node* ld = __ load(__ ctrl(), gc_state, TypeInt::BYTE, T_BYTE, Compile::AliasIdxRaw); marking = __ AndI(ld, __ ConI(ShenandoahHeap::MARKING)); - assert(ShenandoahWriteBarrierNode::is_gc_state_load(ld), "Should match the shape"); + assert(ShenandoahBarrierC2Support::is_gc_state_load(ld), "Should match the shape"); // if (!marking) __ if_then(marking, BoolTest::ne, zero, unlikely); { @@ -361,7 +296,7 @@ bool ShenandoahBarrierSetC2::is_shenandoah_wb_call(Node* call) { return call->is_CallLeaf() && - call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT); + call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT); } bool ShenandoahBarrierSetC2::is_shenandoah_marking_if(PhaseTransform *phase, Node* n) { @@ -549,88 +484,6 @@ return TypeFunc::make(domain, range); } -void ShenandoahBarrierSetC2::resolve_address(C2Access& access) const { - const TypePtr* adr_type = access.addr().type(); - - if ((access.decorators() & IN_NATIVE) == 0 && (adr_type->isa_instptr() || adr_type->isa_aryptr())) { - int off = adr_type->is_ptr()->offset(); - int base_off = adr_type->isa_instptr() ? instanceOopDesc::base_offset_in_bytes() : - arrayOopDesc::base_offset_in_bytes(adr_type->is_aryptr()->elem()->array_element_basic_type()); - assert(off != Type::OffsetTop, "unexpected offset"); - if (off == Type::OffsetBot || off >= base_off) { - DecoratorSet decorators = access.decorators(); - bool is_write = (decorators & C2_WRITE_ACCESS) != 0; - GraphKit* kit = NULL; - if (access.is_parse_access()) { - C2ParseAccess& parse_access = static_cast(access); - kit = parse_access.kit(); - } - Node* adr = access.addr().node(); - assert(adr->is_AddP(), "unexpected address shape"); - Node* base = adr->in(AddPNode::Base); - - if (is_write) { - if (kit != NULL) { - base = shenandoah_write_barrier(kit, base); - } else { - assert(access.is_opt_access(), "either parse or opt access"); - assert((access.decorators() & C2_ARRAY_COPY) != 0, "can be skipped for clone"); - } - } else { - if (adr_type->isa_instptr()) { - Compile* C = access.gvn().C; - ciField* field = C->alias_type(adr_type)->field(); - - // Insert read barrier for Shenandoah. - if (field != NULL && - ((ShenandoahOptimizeStaticFinals && field->is_static() && field->is_final()) || - (ShenandoahOptimizeInstanceFinals && !field->is_static() && field->is_final()) || - (ShenandoahOptimizeStableFinals && field->is_stable()))) { - // Skip the barrier for special fields - } else { - if (kit != NULL) { - base = shenandoah_read_barrier(kit, base); - } else { - assert(access.is_opt_access(), "either parse or opt access"); - assert((access.decorators() & C2_ARRAY_COPY) != 0, "can be skipped for arraycopy"); - } - } - } else { - if (kit != NULL) { - base = shenandoah_read_barrier(kit, base); - } else { - assert(access.is_opt_access(), "either parse or opt access"); - assert((access.decorators() & C2_ARRAY_COPY) != 0, "can be skipped for arraycopy"); - } - } - } - if (base != adr->in(AddPNode::Base)) { - assert(kit != NULL, "no barrier should have been added"); - - Node* address = adr->in(AddPNode::Address); - - if (address->is_AddP()) { - assert(address->in(AddPNode::Base) == adr->in(AddPNode::Base), "unexpected address shape"); - assert(!address->in(AddPNode::Address)->is_AddP(), "unexpected address shape"); - assert(address->in(AddPNode::Address) == adr->in(AddPNode::Base), "unexpected address shape"); - address = address->clone(); - address->set_req(AddPNode::Base, base); - address->set_req(AddPNode::Address, base); - address = kit->gvn().transform(address); - } else { - assert(address == adr->in(AddPNode::Base), "unexpected address shape"); - address = base; - } - adr = adr->clone(); - adr->set_req(AddPNode::Base, base); - adr->set_req(AddPNode::Address, address); - adr = kit->gvn().transform(adr); - access.addr().set_node(adr); - } - } - } -} - Node* ShenandoahBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const { DecoratorSet decorators = access.decorators(); @@ -662,44 +515,8 @@ PhaseGVN& gvn = opt_access.gvn(); MergeMemNode* mm = opt_access.mem(); - if (ShenandoahStoreValReadBarrier) { - RegionNode* region = new RegionNode(3); - const Type* v_t = gvn.type(val.node()); - Node* phi = new PhiNode(region, v_t->isa_oopptr() ? v_t->is_oopptr()->cast_to_nonconst() : v_t); - Node* cmp = gvn.transform(new CmpPNode(val.node(), gvn.zerocon(T_OBJECT))); - Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::ne)); - IfNode* iff = new IfNode(opt_access.ctl(), bol, PROB_LIKELY_MAG(3), COUNT_UNKNOWN); - - gvn.transform(iff); - if (gvn.is_IterGVN()) { - gvn.is_IterGVN()->_worklist.push(iff); - } else { - gvn.record_for_igvn(iff); - } - - Node* null_true = gvn.transform(new IfFalseNode(iff)); - Node* null_false = gvn.transform(new IfTrueNode(iff)); - region->init_req(1, null_true); - region->init_req(2, null_false); - phi->init_req(1, gvn.zerocon(T_OBJECT)); - Node* cast = new CastPPNode(val.node(), gvn.type(val.node())->join_speculative(TypePtr::NOTNULL)); - cast->set_req(0, null_false); - cast = gvn.transform(cast); - Node* rb = gvn.transform(new ShenandoahReadBarrierNode(null_false, gvn.C->immutable_memory(), cast, false)); - phi->init_req(2, rb); - opt_access.set_ctl(gvn.transform(region)); - val.set_node(gvn.transform(phi)); - } if (ShenandoahStoreValEnqueueBarrier) { - const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(gvn.type(val.node())); - int alias = gvn.C->get_alias_index(adr_type); - Node* wb = new ShenandoahWriteBarrierNode(gvn.C, opt_access.ctl(), mm->memory_at(alias), val.node()); - Node* wb_transformed = gvn.transform(wb); - Node* enqueue = gvn.transform(new ShenandoahEnqueueBarrierNode(wb_transformed)); - if (wb_transformed == wb) { - Node* proj = gvn.transform(new ShenandoahWBMemProjNode(wb)); - mm->set_memory_at(alias, proj); - } + Node* enqueue = gvn.transform(new ShenandoahEnqueueBarrierNode(val.node())); val.set_node(enqueue); } } @@ -724,6 +541,17 @@ Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : top; Node* load = BarrierSetC2::load_at_resolved(access, val_type); + if (access.is_oop()) { + if (ShenandoahLoadRefBarrier) { + load = new ShenandoahLoadReferenceBarrierNode(NULL, load); + if (access.is_parse_access()) { + load = static_cast(access).kit()->gvn().transform(load); + } else { + load = static_cast(access).gvn().transform(load); + } + } + } + // If we are reading the value of the referent field of a Reference // object (either by using Unsafe directly or through reflection) // then, if SATB is enabled, we need to record the referent in an @@ -797,9 +625,10 @@ #ifdef _LP64 if (adr->bottom_type()->is_ptr_to_narrowoop()) { - return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type())); + load_store = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type())); } #endif + load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, load_store)); return load_store; } return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type); @@ -867,6 +696,7 @@ } Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type); if (access.is_oop()) { + result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, result)); shenandoah_write_barrier_pre(kit, false /* do_load */, NULL, NULL, max_juint, NULL, NULL, result /* pre_val */, T_OBJECT); @@ -876,19 +706,9 @@ void ShenandoahBarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const { assert(!src->is_AddP(), "unexpected input"); - src = shenandoah_read_barrier(kit, src); BarrierSetC2::clone(kit, src, dst, size, is_array); } -Node* ShenandoahBarrierSetC2::resolve(GraphKit* kit, Node* n, DecoratorSet decorators) const { - bool is_write = decorators & ACCESS_WRITE; - if (is_write) { - return shenandoah_write_barrier(kit, n); - } else { - return shenandoah_read_barrier(kit, n); - } -} - Node* ShenandoahBarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* ctrl, Node* mem, Node* toobig_false, Node* size_in_bytes, Node*& i_o, Node*& needgc_ctrl, Node*& fast_oop_ctrl, Node*& fast_oop_rawmem, @@ -915,6 +735,7 @@ // Support for GC barriers emitted during parsing bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const { + if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) return true; if (node->Opcode() != Op_CallLeaf && node->Opcode() != Op_CallLeafNoFP) { return false; } @@ -929,26 +750,30 @@ } Node* ShenandoahBarrierSetC2::step_over_gc_barrier(Node* c) const { - return ShenandoahBarrierNode::skip_through_barrier(c); + if (c->Opcode() == Op_ShenandoahLoadReferenceBarrier) { + return c->in(ShenandoahLoadReferenceBarrierNode::ValueIn); + } + if (c->Opcode() == Op_ShenandoahEnqueueBarrier) { + c = c->in(1); + } + return c; } bool ShenandoahBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const { - return !ShenandoahWriteBarrierNode::expand(C, igvn); + return !ShenandoahBarrierC2Support::expand(C, igvn); } bool ShenandoahBarrierSetC2::optimize_loops(PhaseIdealLoop* phase, LoopOptsMode mode, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const { if (mode == LoopOptsShenandoahExpand) { assert(UseShenandoahGC, "only for shenandoah"); - ShenandoahWriteBarrierNode::pin_and_expand(phase); + ShenandoahBarrierC2Support::pin_and_expand(phase); return true; } else if (mode == LoopOptsShenandoahPostExpand) { assert(UseShenandoahGC, "only for shenandoah"); visited.Clear(); - ShenandoahWriteBarrierNode::optimize_after_expansion(visited, nstack, worklist, phase); + ShenandoahBarrierC2Support::optimize_after_expansion(visited, nstack, worklist, phase); return true; } - GrowableArray memory_graph_fixers; - ShenandoahWriteBarrierNode::optimize_before_expansion(phase, memory_graph_fixers, false); return false; } @@ -957,7 +782,6 @@ if (!is_oop) { return false; } - if (tightly_coupled_alloc) { if (phase == Optimization) { return false; @@ -985,7 +809,7 @@ } } else { return true; - } + } } else if (src_type->isa_aryptr()) { BasicType src_elem = src_type->klass()->as_array_klass()->element_type()->basic_type(); if (src_elem == T_OBJECT || src_elem == T_ARRAY) { @@ -1038,14 +862,20 @@ // Support for macro expanded GC barriers void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const { - if (node->Opcode() == Op_ShenandoahWriteBarrier) { - state()->add_shenandoah_barrier((ShenandoahWriteBarrierNode*) node); + if (node->Opcode() == Op_ShenandoahEnqueueBarrier) { + state()->add_enqueue_barrier((ShenandoahEnqueueBarrierNode*) node); + } + if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) { + state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node); } } void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const { - if (node->Opcode() == Op_ShenandoahWriteBarrier) { - state()->remove_shenandoah_barrier((ShenandoahWriteBarrierNode*) node); + if (node->Opcode() == Op_ShenandoahEnqueueBarrier) { + state()->remove_enqueue_barrier((ShenandoahEnqueueBarrierNode*) node); + } + if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) { + state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node); } } @@ -1091,19 +921,18 @@ } } } - for (int i = state()->shenandoah_barriers_count()-1; i >= 0; i--) { - ShenandoahWriteBarrierNode* n = state()->shenandoah_barrier(i); + for (int i = state()->enqueue_barriers_count() - 1; i >= 0; i--) { + ShenandoahEnqueueBarrierNode* n = state()->enqueue_barrier(i); if (!useful.member(n)) { - state()->remove_shenandoah_barrier(n); + state()->remove_enqueue_barrier(n); } } - -} - -bool ShenandoahBarrierSetC2::has_special_unique_user(const Node* node) const { - assert(node->outcnt() == 1, "match only for unique out"); - Node* n = node->unique_out(); - return node->Opcode() == Op_ShenandoahWriteBarrier && n->Opcode() == Op_ShenandoahWBMemProj; + for (int i = state()->load_reference_barriers_count() - 1; i >= 0; i--) { + ShenandoahLoadReferenceBarrierNode* n = state()->load_reference_barrier(i); + if (!useful.member(n)) { + state()->remove_load_reference_barrier(n); + } + } } void ShenandoahBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {} @@ -1123,7 +952,7 @@ #ifdef ASSERT void ShenandoahBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const { if (ShenandoahVerifyOptoBarriers && phase == BarrierSetC2::BeforeExpand) { - ShenandoahBarrierNode::verify(Compile::current()->root()); + ShenandoahBarrierC2Support::verify(Compile::current()->root()); } else if (phase == BarrierSetC2::BeforeCodeGen) { // Verify G1 pre-barriers const int marking_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()); @@ -1229,7 +1058,7 @@ } } else if (can_reshape && n->Opcode() == Op_If && - ShenandoahWriteBarrierNode::is_heap_stable_test(n) && + ShenandoahBarrierC2Support::is_heap_stable_test(n) && n->in(0) != NULL) { Node* dom = n->in(0); Node* prev_dom = n; @@ -1237,7 +1066,7 @@ int dist = 16; // Search up the dominator tree for another heap stable test while (dom->Opcode() != op || // Not same opcode? - !ShenandoahWriteBarrierNode::is_heap_stable_test(dom) || // Not same input 1? + !ShenandoahBarrierC2Support::is_heap_stable_test(dom) || // Not same input 1? prev_dom->in(0) != dom) { // One path of test does not dominate? if (dist < 0) return NULL; @@ -1258,46 +1087,6 @@ return NULL; } -Node* ShenandoahBarrierSetC2::identity_node(PhaseGVN* phase, Node* n) const { - if (n->is_Load()) { - Node *mem = n->in(MemNode::Memory); - Node *value = n->as_Load()->can_see_stored_value(mem, phase); - if (value) { - PhaseIterGVN *igvn = phase->is_IterGVN(); - if (igvn != NULL && - value->is_Phi() && - value->req() > 2 && - value->in(1) != NULL && - value->in(1)->is_ShenandoahBarrier()) { - if (igvn->_worklist.member(value) || - igvn->_worklist.member(value->in(0)) || - (value->in(0)->in(1) != NULL && - value->in(0)->in(1)->is_IfProj() && - (igvn->_worklist.member(value->in(0)->in(1)) || - (value->in(0)->in(1)->in(0) != NULL && - igvn->_worklist.member(value->in(0)->in(1)->in(0)))))) { - igvn->_worklist.push(n); - return n; - } - } - // (This works even when value is a Con, but LoadNode::Value - // usually runs first, producing the singleton type of the Con.) - Node *value_no_barrier = step_over_gc_barrier(value->Opcode() == Op_EncodeP ? value->in(1) : value); - if (value->Opcode() == Op_EncodeP) { - if (value_no_barrier != value->in(1)) { - Node *encode = value->clone(); - encode->set_req(1, value_no_barrier); - encode = phase->transform(encode); - return encode; - } - } else { - return value_no_barrier; - } - } - } - return n; -} - bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) { for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { Node* u = n->fast_out(i); @@ -1308,20 +1097,6 @@ return n->outcnt() > 0; } -bool ShenandoahBarrierSetC2::flatten_gc_alias_type(const TypePtr*& adr_type) const { - int offset = adr_type->offset(); - if (offset == ShenandoahBrooksPointer::byte_offset()) { - if (adr_type->isa_aryptr()) { - adr_type = TypeAryPtr::make(adr_type->ptr(), adr_type->isa_aryptr()->ary(), adr_type->isa_aryptr()->klass(), false, offset); - } else if (adr_type->isa_instptr()) { - adr_type = TypeInstPtr::make(adr_type->ptr(), ciEnv::current()->Object_klass(), false, NULL, offset); - } - return true; - } else { - return false; - } -} - bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode) const { switch (opcode) { case Op_CallLeaf: @@ -1356,9 +1131,7 @@ } #endif return true; - case Op_ShenandoahReadBarrier: - return true; - case Op_ShenandoahWriteBarrier: + case Op_ShenandoahLoadReferenceBarrier: assert(false, "should have been expanded already"); return true; default: @@ -1366,17 +1139,6 @@ } } -#ifdef ASSERT -bool ShenandoahBarrierSetC2::verify_gc_alias_type(const TypePtr* adr_type, int offset) const { - if (offset == ShenandoahBrooksPointer::byte_offset() && - (adr_type->base() == Type::AryPtr || adr_type->base() == Type::OopPtr)) { - return true; - } else { - return false; - } -} -#endif - bool ShenandoahBarrierSetC2::escape_add_to_con_graph(ConnectionGraph* conn_graph, PhaseGVN* gvn, Unique_Node_List* delayed_worklist, Node* n, uint opcode) const { switch (opcode) { case Op_ShenandoahCompareAndExchangeP: @@ -1412,15 +1174,12 @@ } return false; } - case Op_ShenandoahReadBarrier: - case Op_ShenandoahWriteBarrier: - // Barriers 'pass through' its arguments. I.e. what goes in, comes out. - // It doesn't escape. - conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahBarrierNode::ValueIn), delayed_worklist); - break; case Op_ShenandoahEnqueueBarrier: conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(1), delayed_worklist); break; + case Op_ShenandoahLoadReferenceBarrier: + conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahLoadReferenceBarrierNode::ValueIn), delayed_worklist); + return true; default: // Nothing break; @@ -1441,15 +1200,12 @@ case Op_ShenandoahWeakCompareAndSwapP: case Op_ShenandoahWeakCompareAndSwapN: return conn_graph->add_final_edges_unsafe_access(n, opcode); - case Op_ShenandoahReadBarrier: - case Op_ShenandoahWriteBarrier: - // Barriers 'pass through' its arguments. I.e. what goes in, comes out. - // It doesn't escape. - conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahBarrierNode::ValueIn), NULL); - return true; case Op_ShenandoahEnqueueBarrier: conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(1), NULL); return true; + case Op_ShenandoahLoadReferenceBarrier: + conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahLoadReferenceBarrierNode::ValueIn), NULL); + return true; default: // Nothing break; @@ -1464,21 +1220,7 @@ } bool ShenandoahBarrierSetC2::escape_is_barrier_node(Node* n) const { - return n->is_ShenandoahBarrier(); -} - -bool ShenandoahBarrierSetC2::matcher_find_shared_visit(Matcher* matcher, Matcher::MStack& mstack, Node* n, uint opcode, bool& mem_op, int& mem_addr_idx) const { - switch (opcode) { - case Op_ShenandoahReadBarrier: - if (n->in(ShenandoahBarrierNode::ValueIn)->is_DecodeNarrowPtr()) { - matcher->set_shared(n->in(ShenandoahBarrierNode::ValueIn)->in(1)); - } - matcher->set_shared(n); - return true; - default: - break; - } - return false; + return n->Opcode() == Op_ShenandoahLoadReferenceBarrier; } bool ShenandoahBarrierSetC2::matcher_find_shared_post_visit(Matcher* matcher, Node* n, uint opcode) const { @@ -1510,62 +1252,3 @@ xop == Op_ShenandoahCompareAndSwapN || xop == Op_ShenandoahCompareAndSwapP; } - -void ShenandoahBarrierSetC2::igvn_add_users_to_worklist(PhaseIterGVN* igvn, Node* use) const { - if (use->is_ShenandoahBarrier()) { - for (DUIterator_Fast i2max, i2 = use->fast_outs(i2max); i2 < i2max; i2++) { - Node* u = use->fast_out(i2); - Node* cmp = use->find_out_with(Op_CmpP); - if (u->Opcode() == Op_CmpP) { - igvn->_worklist.push(cmp); - } - } - } -} - -void ShenandoahBarrierSetC2::ccp_analyze(PhaseCCP* ccp, Unique_Node_List& worklist, Node* use) const { - if (use->is_ShenandoahBarrier()) { - for (DUIterator_Fast i2max, i2 = use->fast_outs(i2max); i2 < i2max; i2++) { - Node* p = use->fast_out(i2); - if (p->Opcode() == Op_AddP) { - for (DUIterator_Fast i3max, i3 = p->fast_outs(i3max); i3 < i3max; i3++) { - Node* q = p->fast_out(i3); - if (q->is_Load()) { - if(q->bottom_type() != ccp->type(q)) { - worklist.push(q); - } - } - } - } - } - } -} - -Node* ShenandoahBarrierSetC2::split_if_pre(PhaseIdealLoop* phase, Node* n) const { - if (n->Opcode() == Op_ShenandoahReadBarrier) { - ((ShenandoahReadBarrierNode*)n)->try_move(phase); - } else if (n->Opcode() == Op_ShenandoahWriteBarrier) { - return ((ShenandoahWriteBarrierNode*)n)->try_split_thru_phi(phase); - } - - return NULL; -} - -bool ShenandoahBarrierSetC2::build_loop_late_post(PhaseIdealLoop* phase, Node* n) const { - return ShenandoahBarrierNode::build_loop_late_post(phase, n); -} - -bool ShenandoahBarrierSetC2::sink_node(PhaseIdealLoop* phase, Node* n, Node* x, Node* x_ctrl, Node* n_ctrl) const { - if (n->is_ShenandoahBarrier()) { - return x->as_ShenandoahBarrier()->sink_node(phase, x_ctrl, n_ctrl); - } - if (n->is_MergeMem()) { - // PhaseIdealLoop::split_if_with_blocks_post() would: - // _igvn._worklist.yank(x); - // which sometimes causes chains of MergeMem which some of - // shenandoah specific code doesn't support - phase->register_new_node(x, x_ctrl); - return true; - } - return false; -}