diff -r 13588c901957 -r 9cf78a70fa4f src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp Thu Oct 17 20:27:44 2019 +0100 +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp Thu Oct 17 20:53:35 2019 +0100 @@ -38,6 +38,7 @@ #include "opto/movenode.hpp" #include "opto/narrowptrnode.hpp" #include "opto/rootnode.hpp" +#include "opto/runtime.hpp" ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() { return reinterpret_cast(BarrierSet::barrier_set()->barrier_set_c2()); @@ -297,7 +298,7 @@ bool ShenandoahBarrierSetC2::is_shenandoah_lrb_call(Node* call) { return call->is_CallLeaf() && - call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT); + call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier); } bool ShenandoahBarrierSetC2::is_shenandoah_marking_if(PhaseTransform *phase, Node* n) { @@ -462,7 +463,7 @@ const TypeFunc* ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type() { const Type **fields = TypeTuple::fields(1); - fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value + fields[TypeFunc::Parms+0] = TypeOopPtr::NOTNULL; // src oop const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields); // create result type (range) @@ -473,9 +474,11 @@ } const TypeFunc* ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type() { - const Type **fields = TypeTuple::fields(1); + const Type **fields = TypeTuple::fields(2); fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value - const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields); + fields[TypeFunc::Parms+1] = TypeRawPtr::BOTTOM; // original load address + + const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields); // create result type (range) fields = TypeTuple::fields(1); @@ -544,7 +547,7 @@ if (access.is_oop()) { if (ShenandoahLoadRefBarrier) { - load = new ShenandoahLoadReferenceBarrierNode(NULL, load); + load = new ShenandoahLoadReferenceBarrierNode(NULL, load, (decorators & IN_NATIVE) != 0); if (access.is_parse_access()) { load = static_cast(access).kit()->gvn().transform(load); } else { @@ -629,7 +632,7 @@ load_store = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type())); } #endif - load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, load_store)); + load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, load_store, false)); return load_store; } return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type); @@ -697,7 +700,7 @@ } Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type); if (access.is_oop()) { - result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, result)); + result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, result, false)); shenandoah_write_barrier_pre(kit, false /* do_load */, NULL, NULL, max_juint, NULL, NULL, result /* pre_val */, T_OBJECT); @@ -705,11 +708,6 @@ return result; } -void ShenandoahBarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const { - assert(!src->is_AddP(), "unexpected input"); - BarrierSetC2::clone(kit, src, dst, size, is_array); -} - // Support for GC barriers emitted during parsing bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const { if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) return true; @@ -755,7 +753,7 @@ } bool ShenandoahBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const { - bool is_oop = type == T_OBJECT || type == T_ARRAY; + bool is_oop = is_reference_type(type); if (!is_oop) { return false; } @@ -771,9 +769,8 @@ return true; } -bool ShenandoahBarrierSetC2::clone_needs_postbarrier(ArrayCopyNode *ac, PhaseIterGVN& igvn) { - Node* src = ac->in(ArrayCopyNode::Src); - const TypeOopPtr* src_type = igvn.type(src)->is_oopptr(); +bool ShenandoahBarrierSetC2::clone_needs_barrier(Node* src, PhaseGVN& gvn) { + const TypeOopPtr* src_type = gvn.type(src)->is_oopptr(); if (src_type->isa_instptr() != NULL) { ciInstanceKlass* ik = src_type->klass()->as_instance_klass(); if ((src_type->klass_is_exact() || (!ik->is_interface() && !ik->has_subklass())) && !ik->has_injected_fields()) { @@ -781,7 +778,7 @@ return true; } else { if (!src_type->klass_is_exact()) { - igvn.C->dependencies()->assert_leaf_type(ik); + Compile::current()->dependencies()->assert_leaf_type(ik); } } } else { @@ -789,7 +786,7 @@ } } else if (src_type->isa_aryptr()) { BasicType src_elem = src_type->klass()->as_array_klass()->element_type()->basic_type(); - if (src_elem == T_OBJECT || src_elem == T_ARRAY) { + if (is_reference_type(src_elem)) { return true; } } else { @@ -798,42 +795,79 @@ return false; } -void ShenandoahBarrierSetC2::clone_barrier_at_expansion(ArrayCopyNode* ac, Node* call, PhaseIterGVN& igvn) const { - assert(ac->is_clonebasic(), "no other kind of arraycopy here"); - - if (!clone_needs_postbarrier(ac, igvn)) { - BarrierSetC2::clone_barrier_at_expansion(ac, call, igvn); - return; - } - - const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; - Node* c = new ProjNode(call,TypeFunc::Control); - c = igvn.transform(c); - Node* m = new ProjNode(call, TypeFunc::Memory); - m = igvn.transform(m); - +void ShenandoahBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const { + Node* ctrl = ac->in(TypeFunc::Control); + Node* mem = ac->in(TypeFunc::Memory); + Node* src = ac->in(ArrayCopyNode::Src); + Node* src_offset = ac->in(ArrayCopyNode::SrcPos); Node* dest = ac->in(ArrayCopyNode::Dest); - assert(dest->is_AddP(), "bad input"); - Node* barrier_call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type(), - CAST_FROM_FN_PTR(address, ShenandoahRuntime::shenandoah_clone_barrier), - "shenandoah_clone_barrier", raw_adr_type); - barrier_call->init_req(TypeFunc::Control, c); - barrier_call->init_req(TypeFunc::I_O , igvn.C->top()); - barrier_call->init_req(TypeFunc::Memory , m); - barrier_call->init_req(TypeFunc::ReturnAdr, igvn.C->top()); - barrier_call->init_req(TypeFunc::FramePtr, igvn.C->top()); - barrier_call->init_req(TypeFunc::Parms+0, dest->in(AddPNode::Base)); + Node* dest_offset = ac->in(ArrayCopyNode::DestPos); + Node* length = ac->in(ArrayCopyNode::Length); + assert (src_offset == NULL && dest_offset == NULL, "for clone offsets should be null"); + assert (src->is_AddP(), "for clone the src should be the interior ptr"); + assert (dest->is_AddP(), "for clone the dst should be the interior ptr"); + + if (ShenandoahCloneBarrier && clone_needs_barrier(src, phase->igvn())) { + // Check if heap is has forwarded objects. If it does, we need to call into the special + // routine that would fix up source references before we can continue. + + enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT }; + Node* region = new RegionNode(PATH_LIMIT); + Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM); + + Node* thread = phase->transform_later(new ThreadLocalNode()); + Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset())); + Node* gc_state_addr = phase->transform_later(new AddPNode(phase->C->top(), thread, offset)); + + uint gc_state_idx = Compile::AliasIdxRaw; + const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument + debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx)); + + Node* gc_state = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered)); + Node* stable_and = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::HAS_FORWARDED))); + Node* stable_cmp = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT))); + Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne)); + + IfNode* stable_iff = phase->transform_later(new IfNode(ctrl, stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN))->as_If(); + Node* stable_ctrl = phase->transform_later(new IfFalseNode(stable_iff)); + Node* unstable_ctrl = phase->transform_later(new IfTrueNode(stable_iff)); - barrier_call = igvn.transform(barrier_call); - c = new ProjNode(barrier_call,TypeFunc::Control); - c = igvn.transform(c); - m = new ProjNode(barrier_call, TypeFunc::Memory); - m = igvn.transform(m); + // Heap is stable, no need to do anything additional + region->init_req(_heap_stable, stable_ctrl); + mem_phi->init_req(_heap_stable, mem); + + // Heap is unstable, call into clone barrier stub + Node* call = phase->make_leaf_call(unstable_ctrl, mem, + ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type(), + CAST_FROM_FN_PTR(address, ShenandoahRuntime::shenandoah_clone_barrier), + "shenandoah_clone", + TypeRawPtr::BOTTOM, + src->in(AddPNode::Base)); + call = phase->transform_later(call); + + ctrl = phase->transform_later(new ProjNode(call, TypeFunc::Control)); + mem = phase->transform_later(new ProjNode(call, TypeFunc::Memory)); + region->init_req(_heap_unstable, ctrl); + mem_phi->init_req(_heap_unstable, mem); - Node* out_c = ac->proj_out(TypeFunc::Control); - Node* out_m = ac->proj_out(TypeFunc::Memory); - igvn.replace_node(out_c, c); - igvn.replace_node(out_m, m); + // Wire up the actual arraycopy stub now + ctrl = phase->transform_later(region); + mem = phase->transform_later(mem_phi); + + const char* name = "arraycopy"; + call = phase->make_leaf_call(ctrl, mem, + OptoRuntime::fast_arraycopy_Type(), + phase->basictype2arraycopy(T_LONG, NULL, NULL, true, name, true), + name, TypeRawPtr::BOTTOM, + src, dest, length + LP64_ONLY(COMMA phase->top())); + call = phase->transform_later(call); + + // Hook up the whole thing into the graph + phase->igvn().replace_node(ac, call); + } else { + BarrierSetC2::clone_at_expansion(phase, ac); + } } @@ -912,8 +946,6 @@ } } -void ShenandoahBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {} - void* ShenandoahBarrierSetC2::create_barrier_state(Arena* comp_arena) const { return new(comp_arena) ShenandoahBarrierSetC2State(comp_arena); } @@ -928,7 +960,7 @@ #ifdef ASSERT void ShenandoahBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const { - if (ShenandoahVerifyOptoBarriers && phase == BarrierSetC2::BeforeExpand) { + if (ShenandoahVerifyOptoBarriers && phase == BarrierSetC2::BeforeMacroExpand) { ShenandoahBarrierC2Support::verify(Compile::current()->root()); } else if (phase == BarrierSetC2::BeforeCodeGen) { // Verify G1 pre-barriers @@ -1196,10 +1228,6 @@ } -bool ShenandoahBarrierSetC2::escape_is_barrier_node(Node* n) const { - return n->Opcode() == Op_ShenandoahLoadReferenceBarrier; -} - bool ShenandoahBarrierSetC2::matcher_find_shared_post_visit(Matcher* matcher, Node* n, uint opcode) const { switch (opcode) { case Op_ShenandoahCompareAndExchangeP: