8221766: Load-reference barriers for Shenandoah
Reviewed-by: kvn, erikj, aph, shade
--- a/make/hotspot/lib/JvmFeatures.gmk Thu Apr 04 07:43:44 2019 -0700
+++ b/make/hotspot/lib/JvmFeatures.gmk Tue Apr 02 23:00:22 2019 +0200
@@ -172,8 +172,6 @@
ifneq ($(call check-jvm-feature, shenandoahgc), true)
JVM_CFLAGS_FEATURES += -DINCLUDE_SHENANDOAHGC=0
JVM_EXCLUDE_PATTERNS += gc/shenandoah
-else
- JVM_CFLAGS_FEATURES += -DSUPPORT_BARRIER_ON_PRIMITIVES -DSUPPORT_NOT_TO_SPACE_INVARIANT
endif
ifneq ($(call check-jvm-feature, jfr), true)
--- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp Tue Apr 02 23:00:22 2019 +0200
@@ -40,7 +40,7 @@
#define __ masm->
-address ShenandoahBarrierSetAssembler::_shenandoah_wb = NULL;
+address ShenandoahBarrierSetAssembler::_shenandoah_lrb = NULL;
void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
Register addr, Register count, RegSet saved_regs) {
@@ -198,60 +198,31 @@
__ bind(done);
}
-void ShenandoahBarrierSetAssembler::read_barrier(MacroAssembler* masm, Register dst) {
- if (ShenandoahReadBarrier) {
- read_barrier_impl(masm, dst);
- }
-}
-
-void ShenandoahBarrierSetAssembler::read_barrier_impl(MacroAssembler* masm, Register dst) {
- assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
+void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst) {
+ assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled");
Label is_null;
__ cbz(dst, is_null);
- read_barrier_not_null_impl(masm, dst);
+ resolve_forward_pointer_not_null(masm, dst);
__ bind(is_null);
}
-void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) {
- if (ShenandoahReadBarrier) {
- read_barrier_not_null_impl(masm, dst);
- }
-}
-
-
-void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) {
- assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
+// IMPORTANT: This must preserve all registers, even rscratch1 and rscratch2.
+void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst) {
+ assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled");
__ ldr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset()));
}
-void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) {
- if (ShenandoahWriteBarrier) {
- write_barrier_impl(masm, dst);
- }
-}
-
-void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) {
- assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "Should be enabled");
- assert(dst != rscratch1, "need rscratch1");
+void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst, Register tmp) {
+ assert(ShenandoahLoadRefBarrier, "Should be enabled");
assert(dst != rscratch2, "need rscratch2");
Label done;
-
+ __ enter();
Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
- __ ldrb(rscratch1, gc_state);
+ __ ldrb(rscratch2, gc_state);
// Check for heap stability
- __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
- __ tst(rscratch1, rscratch2);
- __ br(Assembler::EQ, done);
-
- // Heap is unstable, need to perform the read-barrier even if WB is inactive
- __ ldr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset()));
-
- // Check for evacuation-in-progress and jump to WB slow-path if needed
- __ mov(rscratch2, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
- __ tst(rscratch1, rscratch2);
- __ br(Assembler::EQ, done);
+ __ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
RegSet to_save = RegSet::of(r0);
if (dst != r0) {
@@ -259,7 +230,7 @@
__ mov(r0, dst);
}
- __ far_call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_wb())));
+ __ far_call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_lrb())));
if (dst != r0) {
__ mov(dst, r0);
@@ -267,14 +238,11 @@
}
__ bind(done);
+ __ leave();
}
void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
if (ShenandoahStoreValEnqueueBarrier) {
- Label is_null;
- __ cbz(dst, is_null);
- write_barrier_impl(masm, dst);
- __ bind(is_null);
// Save possibly live regs.
RegSet live_regs = RegSet::range(r0, r4) - dst;
__ push(live_regs, sp);
@@ -286,44 +254,45 @@
__ ldrd(v0, __ post(sp, 2 * wordSize));
__ pop(live_regs, sp);
}
- if (ShenandoahStoreValReadBarrier) {
- read_barrier_impl(masm, dst);
+}
+
+void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Register tmp) {
+ if (ShenandoahLoadRefBarrier) {
+ Label is_null;
+ __ cbz(dst, is_null);
+ load_reference_barrier_not_null(masm, dst, tmp);
+ __ bind(is_null);
}
}
void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register dst, Address src, Register tmp1, Register tmp_thread) {
bool on_oop = type == T_OBJECT || type == T_ARRAY;
- bool in_heap = (decorators & IN_HEAP) != 0;
bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
bool on_reference = on_weak || on_phantom;
- if (in_heap) {
- read_barrier_not_null(masm, src.base());
- }
+ BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
+ if (on_oop) {
+ load_reference_barrier(masm, dst, tmp1);
- BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
- if (ShenandoahKeepAliveBarrier && on_oop && on_reference) {
- __ enter();
- satb_write_barrier_pre(masm /* masm */,
- noreg /* obj */,
- dst /* pre_val */,
- rthread /* thread */,
- tmp1 /* tmp */,
- true /* tosca_live */,
- true /* expand_call */);
- __ leave();
+ if (ShenandoahKeepAliveBarrier && on_reference) {
+ __ enter();
+ satb_write_barrier_pre(masm /* masm */,
+ noreg /* obj */,
+ dst /* pre_val */,
+ rthread /* thread */,
+ tmp1 /* tmp */,
+ true /* tosca_live */,
+ true /* expand_call */);
+ __ leave();
+ }
}
}
void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Address dst, Register val, Register tmp1, Register tmp2) {
bool on_oop = type == T_OBJECT || type == T_ARRAY;
- bool in_heap = (decorators & IN_HEAP) != 0;
- if (in_heap) {
- write_barrier(masm, dst.base());
- }
if (!on_oop) {
BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
return;
@@ -361,21 +330,6 @@
}
-void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register op1, Register op2) {
- __ cmp(op1, op2);
- if (ShenandoahAcmpBarrier) {
- Label done;
- __ br(Assembler::EQ, done);
- // The object may have been evacuated, but we won't see it without a
- // membar here.
- __ membar(Assembler::LoadStore| Assembler::LoadLoad);
- read_barrier(masm, op1);
- read_barrier(masm, op2);
- __ cmp(op1, op2);
- __ bind(done);
- }
-}
-
void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm, Register obj,
Register var_size_in_bytes,
int con_size_in_bytes,
@@ -410,27 +364,6 @@
}
}
-void ShenandoahBarrierSetAssembler::resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj) {
- bool oop_not_null = (decorators & IS_NOT_NULL) != 0;
- bool is_write = (decorators & ACCESS_WRITE) != 0;
- if (is_write) {
- if (oop_not_null) {
- write_barrier(masm, obj);
- } else {
- Label done;
- __ cbz(obj, done);
- write_barrier(masm, obj);
- __ bind(done);
- }
- } else {
- if (oop_not_null) {
- read_barrier_not_null(masm, obj);
- } else {
- read_barrier(masm, obj);
- }
- }
-}
-
void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val,
bool acquire, bool release, bool weak, bool is_cae,
Register result) {
@@ -469,8 +402,8 @@
__ decode_heap_oop(tmp1, tmp1);
__ decode_heap_oop(tmp2, tmp2);
}
- read_barrier_impl(masm, tmp1);
- read_barrier_impl(masm, tmp2);
+ resolve_forward_pointer(masm, tmp1);
+ resolve_forward_pointer(masm, tmp2);
__ cmp(tmp1, tmp2);
// Retry with expected now being the value we just loaded from addr.
__ br(Assembler::EQ, retry);
@@ -515,7 +448,7 @@
__ b(*stub->continuation());
}
-void ShenandoahBarrierSetAssembler::gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub) {
+void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
Register obj = stub->obj()->as_register();
Register res = stub->result()->as_register();
@@ -532,7 +465,7 @@
__ cbz(res, done);
}
- write_barrier(ce->masm(), res);
+ load_reference_barrier_not_null(ce->masm(), res, rscratch1);
__ bind(done);
__ b(*stub->continuation());
@@ -592,14 +525,14 @@
#endif // COMPILER1
-address ShenandoahBarrierSetAssembler::shenandoah_wb() {
- assert(_shenandoah_wb != NULL, "need write barrier stub");
- return _shenandoah_wb;
+address ShenandoahBarrierSetAssembler::shenandoah_lrb() {
+ assert(_shenandoah_lrb != NULL, "need load reference barrier stub");
+ return _shenandoah_lrb;
}
#define __ cgen->assembler()->
-// Shenandoah write barrier.
+// Shenandoah load reference barrier.
//
// Input:
// r0: OOP to evacuate. Not null.
@@ -608,13 +541,13 @@
// r0: Pointer to evacuated OOP.
//
// Trash rscratch1, rscratch2. Preserve everything else.
-address ShenandoahBarrierSetAssembler::generate_shenandoah_wb(StubCodeGenerator* cgen) {
+address ShenandoahBarrierSetAssembler::generate_shenandoah_lrb(StubCodeGenerator* cgen) {
__ align(6);
- StubCodeMark mark(cgen, "StubRoutines", "shenandoah_wb");
+ StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb");
address start = __ pc();
- Label work;
+ Label work, done;
__ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
__ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
__ ldrb(rscratch2, Address(rscratch2, rscratch1));
@@ -622,19 +555,23 @@
__ ret(lr);
__ bind(work);
- Register obj = r0;
+ __ mov(rscratch2, r0);
+ resolve_forward_pointer_not_null(cgen->assembler(), r0);
+ __ cmp(rscratch2, r0);
+ __ br(Assembler::NE, done);
__ enter(); // required for proper stackwalking of RuntimeStub frame
__ push_call_clobbered_registers();
- __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT));
+ __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT));
__ blrt(lr, 1, 0, MacroAssembler::ret_type_integral);
- __ mov(rscratch1, obj);
+ __ mov(rscratch1, r0);
__ pop_call_clobbered_registers();
- __ mov(obj, rscratch1);
+ __ mov(r0, rscratch1);
__ leave(); // required for proper stackwalking of RuntimeStub frame
+ __ bind(done);
__ ret(lr);
return start;
@@ -643,12 +580,12 @@
#undef __
void ShenandoahBarrierSetAssembler::barrier_stubs_init() {
- if (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) {
+ if (ShenandoahLoadRefBarrier) {
int stub_code_size = 2048;
ResourceMark rm;
BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size);
CodeBuffer buf(bb);
StubCodeGenerator cgen(&buf);
- _shenandoah_wb = generate_shenandoah_wb(&cgen);
+ _shenandoah_lrb = generate_shenandoah_lrb(&cgen);
}
}
--- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp Tue Apr 02 23:00:22 2019 +0200
@@ -29,7 +29,7 @@
#ifdef COMPILER1
class LIR_Assembler;
class ShenandoahPreBarrierStub;
-class ShenandoahWriteBarrierStub;
+class ShenandoahLoadReferenceBarrierStub;
class StubAssembler;
class StubCodeGenerator;
#endif
@@ -37,7 +37,7 @@
class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
private:
- static address _shenandoah_wb;
+ static address _shenandoah_lrb;
void satb_write_barrier_pre(MacroAssembler* masm,
Register obj,
@@ -54,24 +54,21 @@
bool tosca_live,
bool expand_call);
- void read_barrier(MacroAssembler* masm, Register dst);
- void read_barrier_impl(MacroAssembler* masm, Register dst);
- void read_barrier_not_null(MacroAssembler* masm, Register dst);
- void read_barrier_not_null_impl(MacroAssembler* masm, Register dst);
- void write_barrier(MacroAssembler* masm, Register dst);
- void write_barrier_impl(MacroAssembler* masm, Register dst);
- void asm_acmp_barrier(MacroAssembler* masm, Register op1, Register op2);
+ void resolve_forward_pointer(MacroAssembler* masm, Register dst);
+ void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst);
+ void load_reference_barrier(MacroAssembler* masm, Register dst, Register tmp);
+ void load_reference_barrier_not_null(MacroAssembler* masm, Register dst, Register tmp);
- address generate_shenandoah_wb(StubCodeGenerator* cgen);
+ address generate_shenandoah_lrb(StubCodeGenerator* cgen);
public:
- static address shenandoah_wb();
+ static address shenandoah_lrb();
void storeval_barrier(MacroAssembler* masm, Register dst, Register tmp);
#ifdef COMPILER1
void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub);
- void gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub);
+ void gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub);
void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
#endif
@@ -83,8 +80,6 @@
Register dst, Address src, Register tmp1, Register tmp_thread);
virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Address dst, Register val, Register tmp1, Register tmp2);
- virtual void obj_equals(MacroAssembler* masm, Register src1, Register src2);
- virtual void resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj);
virtual void tlab_allocate(MacroAssembler* masm, Register obj,
Register var_size_in_bytes,
int con_size_in_bytes,
--- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetC1_aarch64.cpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetC1_aarch64.cpp Tue Apr 02 23:00:22 2019 +0200
@@ -99,6 +99,7 @@
__ xchg(access.resolved_addr(), value_opr, result, tmp);
if (access.is_oop()) {
+ result = load_reference_barrier(access.gen(), result, access.access_emit_info(), true);
if (ShenandoahSATBBarrier) {
pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr,
result /* pre_val */);
--- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoah_aarch64.ad Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoah_aarch64.ad Tue Apr 02 23:00:22 2019 +0200
@@ -45,18 +45,6 @@
%}
%}
-instruct shenandoahRB(iRegPNoSp dst, iRegP src, rFlagsReg cr) %{
- match(Set dst (ShenandoahReadBarrier src));
- format %{ "shenandoah_rb $dst,$src" %}
- ins_encode %{
- Register s = $src$$Register;
- Register d = $dst$$Register;
- __ ldr(d, Address(s, ShenandoahBrooksPointer::byte_offset()));
- %}
- ins_pipe(pipe_class_memory);
-%}
-
-
instruct compareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
match(Set res (ShenandoahCompareAndSwapP mem (Binary oldval newval)));
--- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp Tue Apr 02 23:00:22 2019 +0200
@@ -41,7 +41,7 @@
#define __ masm->
-address ShenandoahBarrierSetAssembler::_shenandoah_wb = NULL;
+address ShenandoahBarrierSetAssembler::_shenandoah_lrb = NULL;
void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register src, Register dst, Register count) {
@@ -293,41 +293,23 @@
__ bind(done);
}
-void ShenandoahBarrierSetAssembler::read_barrier(MacroAssembler* masm, Register dst) {
- if (ShenandoahReadBarrier) {
- read_barrier_impl(masm, dst);
- }
-}
-
-void ShenandoahBarrierSetAssembler::read_barrier_impl(MacroAssembler* masm, Register dst) {
- assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
+void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst) {
+ assert(ShenandoahCASBarrier, "should be enabled");
Label is_null;
__ testptr(dst, dst);
__ jcc(Assembler::zero, is_null);
- read_barrier_not_null_impl(masm, dst);
+ resolve_forward_pointer_not_null(masm, dst);
__ bind(is_null);
}
-void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) {
- if (ShenandoahReadBarrier) {
- read_barrier_not_null_impl(masm, dst);
- }
-}
-
-void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) {
- assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
+void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst) {
+ assert(ShenandoahCASBarrier || ShenandoahLoadRefBarrier, "should be enabled");
__ movptr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset()));
}
-void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) {
- if (ShenandoahWriteBarrier) {
- write_barrier_impl(masm, dst);
- }
-}
-
-void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) {
- assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "Should be enabled");
+void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst) {
+ assert(ShenandoahLoadRefBarrier, "Should be enabled");
#ifdef _LP64
Label done;
@@ -335,8 +317,8 @@
__ testb(gc_state, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
__ jccb(Assembler::zero, done);
- // Heap is unstable, need to perform the read-barrier even if WB is inactive
- read_barrier_not_null(masm, dst);
+ // Heap is unstable, need to perform the resolve even if LRB is inactive
+ resolve_forward_pointer_not_null(masm, dst);
__ testb(gc_state, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
__ jccb(Assembler::zero, done);
@@ -345,7 +327,7 @@
__ xchgptr(dst, rax); // Move obj into rax and save rax into obj.
}
- __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_wb())));
+ __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_lrb())));
if (dst != rax) {
__ xchgptr(rax, dst); // Swap back obj with rax.
@@ -358,24 +340,18 @@
}
void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
- if (ShenandoahStoreValReadBarrier || ShenandoahStoreValEnqueueBarrier) {
+ if (ShenandoahStoreValEnqueueBarrier) {
storeval_barrier_impl(masm, dst, tmp);
}
}
void ShenandoahBarrierSetAssembler::storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp) {
- assert(UseShenandoahGC && (ShenandoahStoreValReadBarrier || ShenandoahStoreValEnqueueBarrier), "should be enabled");
+ assert(ShenandoahStoreValEnqueueBarrier, "should be enabled");
if (dst == noreg) return;
#ifdef _LP64
if (ShenandoahStoreValEnqueueBarrier) {
- Label is_null;
- __ testptr(dst, dst);
- __ jcc(Assembler::zero, is_null);
- write_barrier_impl(masm, dst);
- __ bind(is_null);
-
// The set of registers to be saved+restored is the same as in the write-barrier above.
// Those are the commonly used registers in the interpreter.
__ pusha();
@@ -389,50 +365,54 @@
//__ pop_callee_saved_registers();
__ popa();
}
- if (ShenandoahStoreValReadBarrier) {
- read_barrier_impl(masm, dst);
- }
#else
Unimplemented();
#endif
}
+void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst) {
+ if (ShenandoahLoadRefBarrier) {
+ Label done;
+ __ testptr(dst, dst);
+ __ jcc(Assembler::zero, done);
+ load_reference_barrier_not_null(masm, dst);
+ __ bind(done);
+ }
+}
+
void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register dst, Address src, Register tmp1, Register tmp_thread) {
bool on_oop = type == T_OBJECT || type == T_ARRAY;
- bool in_heap = (decorators & IN_HEAP) != 0;
bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
bool on_reference = on_weak || on_phantom;
- if (in_heap) {
- read_barrier_not_null(masm, src.base());
- }
- BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
- if (ShenandoahKeepAliveBarrier && on_oop && on_reference) {
- const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
- NOT_LP64(__ get_thread(thread));
+ BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
+ if (on_oop) {
+ load_reference_barrier(masm, dst);
- // Generate the SATB pre-barrier code to log the value of
- // the referent field in an SATB buffer.
- shenandoah_write_barrier_pre(masm /* masm */,
- noreg /* obj */,
- dst /* pre_val */,
- thread /* thread */,
- tmp1 /* tmp */,
- true /* tosca_live */,
- true /* expand_call */);
+ if (ShenandoahKeepAliveBarrier && on_reference) {
+ const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
+ NOT_LP64(__ get_thread(thread));
+ // Generate the SATB pre-barrier code to log the value of
+ // the referent field in an SATB buffer.
+ shenandoah_write_barrier_pre(masm /* masm */,
+ noreg /* obj */,
+ dst /* pre_val */,
+ thread /* thread */,
+ tmp1 /* tmp */,
+ true /* tosca_live */,
+ true /* expand_call */);
+ }
}
}
void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Address dst, Register val, Register tmp1, Register tmp2) {
+ bool on_oop = type == T_OBJECT || type == T_ARRAY;
bool in_heap = (decorators & IN_HEAP) != 0;
bool as_normal = (decorators & AS_NORMAL) != 0;
- if (in_heap) {
- write_barrier(masm, dst.base());
- }
- if (type == T_OBJECT || type == T_ARRAY) {
+ if (on_oop && in_heap) {
bool needs_pre_barrier = as_normal;
Register tmp3 = LP64_ONLY(r8) NOT_LP64(rsi);
@@ -475,44 +455,6 @@
}
}
-#ifndef _LP64
-void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm,
- Address obj1, jobject obj2) {
- Unimplemented();
-}
-
-void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm,
- Register obj1, jobject obj2) {
- Unimplemented();
-}
-#endif
-
-
-void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register op1, Register op2) {
- __ cmpptr(op1, op2);
- if (ShenandoahAcmpBarrier) {
- Label done;
- __ jccb(Assembler::equal, done);
- read_barrier(masm, op1);
- read_barrier(masm, op2);
- __ cmpptr(op1, op2);
- __ bind(done);
- }
-}
-
-void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register src1, Address src2) {
- __ cmpptr(src1, src2);
- if (ShenandoahAcmpBarrier) {
- Label done;
- __ jccb(Assembler::equal, done);
- __ movptr(rscratch2, src2);
- read_barrier(masm, src1);
- read_barrier(masm, rscratch2);
- __ cmpptr(src1, rscratch2);
- __ bind(done);
- }
-}
-
void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm,
Register thread, Register obj,
Register var_size_in_bytes,
@@ -562,28 +504,6 @@
__ verify_tlab();
}
-void ShenandoahBarrierSetAssembler::resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj) {
- bool oop_not_null = (decorators & IS_NOT_NULL) != 0;
- bool is_write = (decorators & ACCESS_WRITE) != 0;
- if (is_write) {
- if (oop_not_null) {
- write_barrier(masm, obj);
- } else {
- Label done;
- __ testptr(obj, obj);
- __ jcc(Assembler::zero, done);
- write_barrier(masm, obj);
- __ bind(done);
- }
- } else {
- if (oop_not_null) {
- read_barrier_not_null(masm, obj);
- } else {
- read_barrier(masm, obj);
- }
- }
-}
-
// Special Shenandoah CAS implementation that handles false negatives
// due to concurrent evacuation.
#ifndef _LP64
@@ -622,14 +542,14 @@
// Step 2. CAS had failed. This may be a false negative.
//
// The trouble comes when we compare the to-space pointer with the from-space
- // pointer to the same object. To resolve this, it will suffice to read both
- // oldval and the value from memory through the read barriers -- this will give
- // both to-space pointers. If they mismatch, then it was a legitimate failure.
+ // pointer to the same object. To resolve this, it will suffice to resolve both
+ // oldval and the value from memory -- this will give both to-space pointers.
+ // If they mismatch, then it was a legitimate failure.
//
if (UseCompressedOops) {
__ decode_heap_oop(tmp1);
}
- read_barrier_impl(masm, tmp1);
+ resolve_forward_pointer(masm, tmp1);
if (UseCompressedOops) {
__ movl(tmp2, oldval);
@@ -637,7 +557,7 @@
} else {
__ movptr(tmp2, oldval);
}
- read_barrier_impl(masm, tmp2);
+ resolve_forward_pointer(masm, tmp2);
__ cmpptr(tmp1, tmp2);
__ jcc(Assembler::notEqual, done, true);
@@ -646,8 +566,8 @@
//
// Corner case: it may happen that somebody stored the from-space pointer
// to memory while we were preparing for retry. Therefore, we can fail again
- // on retry, and so need to do this in loop, always re-reading the failure
- // witness through the read barrier.
+ // on retry, and so need to do this in loop, always resolving the failure
+ // witness.
__ bind(retry);
if (os::is_MP()) __ lock();
if (UseCompressedOops) {
@@ -663,7 +583,7 @@
} else {
__ movptr(tmp2, oldval);
}
- read_barrier_impl(masm, tmp2);
+ resolve_forward_pointer(masm, tmp2);
__ cmpptr(tmp1, tmp2);
__ jcc(Assembler::equal, retry, true);
@@ -811,7 +731,7 @@
}
-void ShenandoahBarrierSetAssembler::gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub) {
+void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
__ bind(*stub->entry());
Label done;
@@ -828,7 +748,7 @@
__ jcc(Assembler::zero, done);
}
- write_barrier(ce->masm(), res);
+ load_reference_barrier_not_null(ce->masm(), res);
__ bind(done);
__ jmp(*stub->continuation());
@@ -898,16 +818,16 @@
#endif // COMPILER1
-address ShenandoahBarrierSetAssembler::shenandoah_wb() {
- assert(_shenandoah_wb != NULL, "need write barrier stub");
- return _shenandoah_wb;
+address ShenandoahBarrierSetAssembler::shenandoah_lrb() {
+ assert(_shenandoah_lrb != NULL, "need load reference barrier stub");
+ return _shenandoah_lrb;
}
#define __ cgen->assembler()->
-address ShenandoahBarrierSetAssembler::generate_shenandoah_wb(StubCodeGenerator* cgen) {
+address ShenandoahBarrierSetAssembler::generate_shenandoah_lrb(StubCodeGenerator* cgen) {
__ align(CodeEntryAlignment);
- StubCodeMark mark(cgen, "StubRoutines", "shenandoah_wb");
+ StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb");
address start = __ pc();
#ifdef _LP64
@@ -955,7 +875,7 @@
__ push(r15);
save_vector_registers(cgen->assembler());
__ movptr(rdi, rax);
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT), rdi);
+ __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT), rdi);
restore_vector_registers(cgen->assembler());
__ pop(r15);
__ pop(r14);
@@ -982,12 +902,12 @@
#undef __
void ShenandoahBarrierSetAssembler::barrier_stubs_init() {
- if (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) {
+ if (ShenandoahLoadRefBarrier) {
int stub_code_size = 4096;
ResourceMark rm;
BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size);
CodeBuffer buf(bb);
StubCodeGenerator cgen(&buf);
- _shenandoah_wb = generate_shenandoah_wb(&cgen);
+ _shenandoah_lrb = generate_shenandoah_lrb(&cgen);
}
}
--- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp Tue Apr 02 23:00:22 2019 +0200
@@ -29,7 +29,7 @@
#ifdef COMPILER1
class LIR_Assembler;
class ShenandoahPreBarrierStub;
-class ShenandoahWriteBarrierStub;
+class ShenandoahLoadReferenceBarrierStub;
class StubAssembler;
class StubCodeGenerator;
#endif
@@ -37,7 +37,7 @@
class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
private:
- static address _shenandoah_wb;
+ static address _shenandoah_lrb;
void satb_write_barrier_pre(MacroAssembler* masm,
Register obj,
@@ -55,32 +55,30 @@
bool tosca_live,
bool expand_call);
- void read_barrier(MacroAssembler* masm, Register dst);
- void read_barrier_impl(MacroAssembler* masm, Register dst);
+ void resolve_forward_pointer(MacroAssembler* masm, Register dst);
+ void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst);
- void read_barrier_not_null(MacroAssembler* masm, Register dst);
- void read_barrier_not_null_impl(MacroAssembler* masm, Register dst);
-
- void write_barrier(MacroAssembler* masm, Register dst);
- void write_barrier_impl(MacroAssembler* masm, Register dst);
+ void load_reference_barrier_not_null(MacroAssembler* masm, Register dst);
void storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp);
- address generate_shenandoah_wb(StubCodeGenerator* cgen);
+ address generate_shenandoah_lrb(StubCodeGenerator* cgen);
void save_vector_registers(MacroAssembler* masm);
void restore_vector_registers(MacroAssembler* masm);
public:
- static address shenandoah_wb();
+ static address shenandoah_lrb();
void storeval_barrier(MacroAssembler* masm, Register dst, Register tmp);
#ifdef COMPILER1
void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub);
- void gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub);
+ void gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub);
void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
#endif
+ void load_reference_barrier(MacroAssembler* masm, Register dst);
+
void cmpxchg_oop(MacroAssembler* masm,
Register res, Address addr, Register oldval, Register newval,
bool exchange, Register tmp1, Register tmp2);
@@ -93,16 +91,6 @@
virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Address dst, Register val, Register tmp1, Register tmp2);
-#ifndef _LP64
- virtual void obj_equals(MacroAssembler* masm,
- Address obj1, jobject obj2);
- virtual void obj_equals(MacroAssembler* masm,
- Register obj1, jobject obj2);
-#endif
-
- virtual void obj_equals(MacroAssembler* masm, Register src1, Register src2);
- virtual void obj_equals(MacroAssembler* masm, Register src1, Address src2);
-
virtual void tlab_allocate(MacroAssembler* masm,
Register thread, Register obj,
Register var_size_in_bytes,
@@ -110,8 +98,6 @@
Register t1, Register t2,
Label& slow_case);
- virtual void resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj);
-
virtual void barrier_stubs_init();
};
--- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetC1_x86.cpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetC1_x86.cpp Tue Apr 02 23:00:22 2019 +0200
@@ -107,6 +107,7 @@
__ xchg(access.resolved_addr(), result, result, LIR_OprFact::illegalOpr);
if (access.is_oop()) {
+ result = load_reference_barrier(access.gen(), result, access.access_emit_info(), true);
if (ShenandoahSATBBarrier) {
pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr,
result /* pre_val */);
--- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoah_x86_64.ad Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoah_x86_64.ad Tue Apr 02 23:00:22 2019 +0200
@@ -23,47 +23,7 @@
source_hpp %{
#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
-%}
-
-instruct shenandoahRB(rRegP dst, rRegP src, rFlagsReg cr) %{
- match(Set dst (ShenandoahReadBarrier src));
- effect(DEF dst, USE src);
- ins_cost(125); // XXX
- format %{ "shenandoah_rb $dst, $src" %}
- ins_encode %{
- Register d = $dst$$Register;
- Register s = $src$$Register;
- __ movptr(d, Address(s, ShenandoahBrooksPointer::byte_offset()));
- %}
- ins_pipe(ialu_reg_mem);
-%}
-
-instruct shenandoahRBNarrow(rRegP dst, rRegN src) %{
- predicate(UseCompressedOops && (Universe::narrow_oop_shift() == 0));
- match(Set dst (ShenandoahReadBarrier (DecodeN src)));
- effect(DEF dst, USE src);
- ins_cost(125); // XXX
- format %{ "shenandoah_rb $dst, $src" %}
- ins_encode %{
- Register d = $dst$$Register;
- Register s = $src$$Register;
- __ movptr(d, Address(r12, s, Address::times_1, ShenandoahBrooksPointer::byte_offset()));
- %}
- ins_pipe(ialu_reg_mem);
-%}
-
-instruct shenandoahRBNarrowShift(rRegP dst, rRegN src) %{
- predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8));
- match(Set dst (ShenandoahReadBarrier (DecodeN src)));
- effect(DEF dst, USE src);
- ins_cost(125); // XXX
- format %{ "shenandoah_rb $dst, $src" %}
- ins_encode %{
- Register d = $dst$$Register;
- Register s = $src$$Register;
- __ movptr(d, Address(r12, s, Address::times_8, ShenandoahBrooksPointer::byte_offset()));
- %}
- ins_pipe(ialu_reg_mem);
+#include "gc/shenandoah/c2/shenandoahSupport.hpp"
%}
instruct compareAndSwapP_shenandoah(rRegI res,
--- a/src/hotspot/share/adlc/formssel.cpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/adlc/formssel.cpp Tue Apr 02 23:00:22 2019 +0200
@@ -777,8 +777,7 @@
!strcmp(_matrule->_rChild->_opType,"CompareAndExchangeP") ||
!strcmp(_matrule->_rChild->_opType,"CompareAndExchangeN") ||
!strcmp(_matrule->_rChild->_opType,"ShenandoahCompareAndExchangeP") ||
- !strcmp(_matrule->_rChild->_opType,"ShenandoahCompareAndExchangeN") ||
- !strcmp(_matrule->_rChild->_opType,"ShenandoahReadBarrier"))) return true;
+ !strcmp(_matrule->_rChild->_opType,"ShenandoahCompareAndExchangeN"))) return true;
else if ( is_ideal_load() == Form::idealP ) return true;
else if ( is_ideal_store() != Form::none ) return true;
@@ -3506,7 +3505,6 @@
"ClearArray",
"GetAndSetB", "GetAndSetS", "GetAndAddI", "GetAndSetI", "GetAndSetP",
"GetAndAddB", "GetAndAddS", "GetAndAddL", "GetAndSetL", "GetAndSetN",
- "ShenandoahReadBarrier",
"LoadBarrierSlowReg", "LoadBarrierWeakSlowReg"
};
int cnt = sizeof(needs_ideal_memory_list)/sizeof(char*);
--- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp Tue Apr 02 23:00:22 2019 +0200
@@ -46,9 +46,9 @@
bs->gen_pre_barrier_stub(ce, this);
}
-void ShenandoahWriteBarrierStub::emit_code(LIR_Assembler* ce) {
+void ShenandoahLoadReferenceBarrierStub::emit_code(LIR_Assembler* ce) {
ShenandoahBarrierSetAssembler* bs = (ShenandoahBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
- bs->gen_write_barrier_stub(ce, this);
+ bs->gen_load_reference_barrier_stub(ce, this);
}
void ShenandoahBarrierSetC1::pre_barrier(LIRGenerator* gen, CodeEmitInfo* info, DecoratorSet decorators, LIR_Opr addr_opr, LIR_Opr pre_val) {
@@ -105,40 +105,16 @@
__ branch_destination(slow->continuation());
}
-LIR_Opr ShenandoahBarrierSetC1::read_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) {
- if (UseShenandoahGC && ShenandoahReadBarrier) {
- return read_barrier_impl(gen, obj, info, need_null_check);
+LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) {
+ if (ShenandoahLoadRefBarrier) {
+ return load_reference_barrier_impl(gen, obj, info, need_null_check);
} else {
return obj;
}
}
-LIR_Opr ShenandoahBarrierSetC1::read_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) {
- assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier), "Should be enabled");
- LabelObj* done = new LabelObj();
- LIR_Opr result = gen->new_register(T_OBJECT);
- __ move(obj, result);
- if (need_null_check) {
- __ cmp(lir_cond_equal, result, LIR_OprFact::oopConst(NULL));
- __ branch(lir_cond_equal, T_LONG, done->label());
- }
- LIR_Address* brooks_ptr_address = gen->generate_address(result, ShenandoahBrooksPointer::byte_offset(), T_ADDRESS);
- __ load(brooks_ptr_address, result, info ? new CodeEmitInfo(info) : NULL, lir_patch_none);
-
- __ branch_destination(done->label());
- return result;
-}
-
-LIR_Opr ShenandoahBarrierSetC1::write_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) {
- if (UseShenandoahGC && ShenandoahWriteBarrier) {
- return write_barrier_impl(gen, obj, info, need_null_check);
- } else {
- return obj;
- }
-}
-
-LIR_Opr ShenandoahBarrierSetC1::write_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) {
- assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "Should be enabled");
+LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) {
+ assert(ShenandoahLoadRefBarrier, "Should be enabled");
obj = ensure_in_register(gen, obj);
assert(obj->is_register(), "must be a register at this point");
@@ -168,7 +144,7 @@
}
__ cmp(lir_cond_notEqual, flag_val, LIR_OprFact::intConst(0));
- CodeStub* slow = new ShenandoahWriteBarrierStub(obj, result, info ? new CodeEmitInfo(info) : NULL, need_null_check);
+ CodeStub* slow = new ShenandoahLoadReferenceBarrierStub(obj, result, info ? new CodeEmitInfo(info) : NULL, need_null_check);
__ branch(lir_cond_notEqual, T_INT, slow);
__ branch_destination(slow->continuation());
@@ -189,58 +165,13 @@
}
LIR_Opr ShenandoahBarrierSetC1::storeval_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, DecoratorSet decorators) {
- bool need_null_check = (decorators & IS_NOT_NULL) == 0;
if (ShenandoahStoreValEnqueueBarrier) {
- obj = write_barrier_impl(gen, obj, info, need_null_check);
+ obj = ensure_in_register(gen, obj);
pre_barrier(gen, info, decorators, LIR_OprFact::illegalOpr, obj);
}
- if (ShenandoahStoreValReadBarrier) {
- obj = read_barrier_impl(gen, obj, info, true /*need_null_check*/);
- }
return obj;
}
-LIR_Opr ShenandoahBarrierSetC1::resolve_address(LIRAccess& access, bool resolve_in_register) {
- DecoratorSet decorators = access.decorators();
- bool is_array = (decorators & IS_ARRAY) != 0;
- bool needs_patching = (decorators & C1_NEEDS_PATCHING) != 0;
-
- bool is_write = (decorators & ACCESS_WRITE) != 0;
- bool needs_null_check = (decorators & IS_NOT_NULL) == 0;
-
- LIR_Opr base = access.base().item().result();
- LIR_Opr offset = access.offset().opr();
- LIRGenerator* gen = access.gen();
-
- if (is_write) {
- base = write_barrier(gen, base, access.access_emit_info(), needs_null_check);
- } else {
- base = read_barrier(gen, base, access.access_emit_info(), needs_null_check);
- }
-
- LIR_Opr addr_opr;
- if (is_array) {
- addr_opr = LIR_OprFact::address(gen->emit_array_address(base, offset, access.type()));
- } else if (needs_patching) {
- // we need to patch the offset in the instruction so don't allow
- // generate_address to try to be smart about emitting the -1.
- // Otherwise the patching code won't know how to find the
- // instruction to patch.
- addr_opr = LIR_OprFact::address(new LIR_Address(base, PATCHED_ADDR, access.type()));
- } else {
- addr_opr = LIR_OprFact::address(gen->generate_address(base, offset, 0, 0, access.type()));
- }
-
- if (resolve_in_register) {
- LIR_Opr resolved_addr = gen->new_pointer_register();
- __ leal(addr_opr, resolved_addr);
- resolved_addr = LIR_OprFact::address(new LIR_Address(resolved_addr, access.type()));
- return resolved_addr;
- } else {
- return addr_opr;
- }
-}
-
void ShenandoahBarrierSetC1::store_at_resolved(LIRAccess& access, LIR_Opr value) {
if (access.is_oop()) {
if (ShenandoahSATBBarrier) {
@@ -252,15 +183,28 @@
}
void ShenandoahBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) {
- BarrierSetC1::load_at_resolved(access, result);
+ if (!access.is_oop()) {
+ BarrierSetC1::load_at_resolved(access, result);
+ return;
+ }
+
+ LIRGenerator *gen = access.gen();
+
+ if (ShenandoahLoadRefBarrier) {
+ LIR_Opr tmp = gen->new_register(T_OBJECT);
+ BarrierSetC1::load_at_resolved(access, tmp);
+ tmp = load_reference_barrier(access.gen(), tmp, access.access_emit_info(), true);
+ __ move(tmp, result);
+ } else {
+ BarrierSetC1::load_at_resolved(access, result);
+ }
if (ShenandoahKeepAliveBarrier) {
DecoratorSet decorators = access.decorators();
bool is_weak = (decorators & ON_WEAK_OOP_REF) != 0;
bool is_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
bool is_anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
- LIRGenerator *gen = access.gen();
- if (access.is_oop() && (is_weak || is_phantom || is_anonymous)) {
+ if (is_weak || is_phantom || is_anonymous) {
// Register the value in the referent field with the pre-barrier
LabelObj *Lcont_anonymous;
if (is_anonymous) {
@@ -276,19 +220,6 @@
}
}
-LIR_Opr ShenandoahBarrierSetC1::atomic_add_at_resolved(LIRAccess& access, LIRItem& value) {
- return BarrierSetC1::atomic_add_at_resolved(access, value);
-}
-
-LIR_Opr ShenandoahBarrierSetC1::resolve(LIRGenerator* gen, DecoratorSet decorators, LIR_Opr obj) {
- bool is_write = decorators & ACCESS_WRITE;
- if (is_write) {
- return write_barrier(gen, obj, NULL, (decorators & IS_NOT_NULL) == 0);
- } else {
- return read_barrier(gen, obj, NULL, (decorators & IS_NOT_NULL) == 0);
- }
-}
-
class C1ShenandoahPreBarrierCodeGenClosure : public StubAssemblerCodeGenClosure {
virtual OopMapSet* generate_code(StubAssembler* sasm) {
ShenandoahBarrierSetAssembler* bs = (ShenandoahBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
--- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp Tue Apr 02 23:00:22 2019 +0200
@@ -85,7 +85,7 @@
#endif // PRODUCT
};
-class ShenandoahWriteBarrierStub: public CodeStub {
+class ShenandoahLoadReferenceBarrierStub: public CodeStub {
friend class ShenandoahBarrierSetC1;
private:
LIR_Opr _obj;
@@ -94,7 +94,7 @@
bool _needs_null_check;
public:
- ShenandoahWriteBarrierStub(LIR_Opr obj, LIR_Opr result, CodeEmitInfo* info, bool needs_null_check) :
+ ShenandoahLoadReferenceBarrierStub(LIR_Opr obj, LIR_Opr result, CodeEmitInfo* info, bool needs_null_check) :
_obj(obj), _result(result), _info(info), _needs_null_check(needs_null_check)
{
assert(_obj->is_register(), "should be register");
@@ -113,7 +113,7 @@
visitor->do_temp(_result);
}
#ifndef PRODUCT
- virtual void print_name(outputStream* out) const { out->print("ShenandoahWritePreBarrierStub"); }
+ virtual void print_name(outputStream* out) const { out->print("ShenandoahLoadReferenceBarrierStub"); }
#endif // PRODUCT
};
@@ -181,12 +181,10 @@
void pre_barrier(LIRGenerator* gen, CodeEmitInfo* info, DecoratorSet decorators, LIR_Opr addr_opr, LIR_Opr pre_val);
- LIR_Opr read_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check);
- LIR_Opr write_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check);
+ LIR_Opr load_reference_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check);
LIR_Opr storeval_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, DecoratorSet decorators);
- LIR_Opr read_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check);
- LIR_Opr write_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check);
+ LIR_Opr load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check);
LIR_Opr ensure_in_register(LIRGenerator* gen, LIR_Opr obj);
@@ -194,7 +192,6 @@
CodeBlob* pre_barrier_c1_runtime_code_blob() { return _pre_barrier_c1_runtime_code_blob; }
protected:
- virtual LIR_Opr resolve_address(LIRAccess& access, bool resolve_in_register);
virtual void store_at_resolved(LIRAccess& access, LIR_Opr value);
virtual void load_at_resolved(LIRAccess& access, LIR_Opr result);
@@ -202,10 +199,8 @@
virtual LIR_Opr atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value);
virtual LIR_Opr atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value);
- virtual LIR_Opr atomic_add_at_resolved(LIRAccess& access, LIRItem& value);
public:
- virtual LIR_Opr resolve(LIRGenerator* gen, DecoratorSet decorators, LIR_Opr obj);
virtual void generate_c1_runtime_stubs(BufferBlob* buffer_blob);
};
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp Tue Apr 02 23:00:22 2019 +0200
@@ -43,121 +43,56 @@
}
ShenandoahBarrierSetC2State::ShenandoahBarrierSetC2State(Arena* comp_arena)
- : _shenandoah_barriers(new (comp_arena) GrowableArray<ShenandoahWriteBarrierNode*>(comp_arena, 8, 0, NULL)) {
+ : _enqueue_barriers(new (comp_arena) GrowableArray<ShenandoahEnqueueBarrierNode*>(comp_arena, 8, 0, NULL)),
+ _load_reference_barriers(new (comp_arena) GrowableArray<ShenandoahLoadReferenceBarrierNode*>(comp_arena, 8, 0, NULL)) {
}
-int ShenandoahBarrierSetC2State::shenandoah_barriers_count() const {
- return _shenandoah_barriers->length();
+int ShenandoahBarrierSetC2State::enqueue_barriers_count() const {
+ return _enqueue_barriers->length();
}
-ShenandoahWriteBarrierNode* ShenandoahBarrierSetC2State::shenandoah_barrier(int idx) const {
- return _shenandoah_barriers->at(idx);
+ShenandoahEnqueueBarrierNode* ShenandoahBarrierSetC2State::enqueue_barrier(int idx) const {
+ return _enqueue_barriers->at(idx);
}
-void ShenandoahBarrierSetC2State::add_shenandoah_barrier(ShenandoahWriteBarrierNode * n) {
- assert(!_shenandoah_barriers->contains(n), "duplicate entry in barrier list");
- _shenandoah_barriers->append(n);
+void ShenandoahBarrierSetC2State::add_enqueue_barrier(ShenandoahEnqueueBarrierNode * n) {
+ assert(!_enqueue_barriers->contains(n), "duplicate entry in barrier list");
+ _enqueue_barriers->append(n);
}
-void ShenandoahBarrierSetC2State::remove_shenandoah_barrier(ShenandoahWriteBarrierNode * n) {
- if (_shenandoah_barriers->contains(n)) {
- _shenandoah_barriers->remove(n);
+void ShenandoahBarrierSetC2State::remove_enqueue_barrier(ShenandoahEnqueueBarrierNode * n) {
+ if (_enqueue_barriers->contains(n)) {
+ _enqueue_barriers->remove(n);
}
}
-#define __ kit->
+int ShenandoahBarrierSetC2State::load_reference_barriers_count() const {
+ return _load_reference_barriers->length();
+}
+
+ShenandoahLoadReferenceBarrierNode* ShenandoahBarrierSetC2State::load_reference_barrier(int idx) const {
+ return _load_reference_barriers->at(idx);
+}
-Node* ShenandoahBarrierSetC2::shenandoah_read_barrier(GraphKit* kit, Node* obj) const {
- if (ShenandoahReadBarrier) {
- obj = shenandoah_read_barrier_impl(kit, obj, false, true, true);
+void ShenandoahBarrierSetC2State::add_load_reference_barrier(ShenandoahLoadReferenceBarrierNode * n) {
+ assert(!_load_reference_barriers->contains(n), "duplicate entry in barrier list");
+ _load_reference_barriers->append(n);
+}
+
+void ShenandoahBarrierSetC2State::remove_load_reference_barrier(ShenandoahLoadReferenceBarrierNode * n) {
+ if (_load_reference_barriers->contains(n)) {
+ _load_reference_barriers->remove(n);
}
- return obj;
}
Node* ShenandoahBarrierSetC2::shenandoah_storeval_barrier(GraphKit* kit, Node* obj) const {
if (ShenandoahStoreValEnqueueBarrier) {
- obj = shenandoah_write_barrier(kit, obj);
obj = shenandoah_enqueue_barrier(kit, obj);
}
- if (ShenandoahStoreValReadBarrier) {
- obj = shenandoah_read_barrier_impl(kit, obj, true, false, false);
- }
return obj;
}
-Node* ShenandoahBarrierSetC2::shenandoah_read_barrier_impl(GraphKit* kit, Node* obj, bool use_ctrl, bool use_mem, bool allow_fromspace) const {
- const Type* obj_type = obj->bottom_type();
- if (obj_type->higher_equal(TypePtr::NULL_PTR)) {
- return obj;
- }
- const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type);
- Node* mem = use_mem ? __ memory(adr_type) : __ immutable_memory();
-
- if (! ShenandoahBarrierNode::needs_barrier(&__ gvn(), NULL, obj, mem, allow_fromspace)) {
- // We know it is null, no barrier needed.
- return obj;
- }
-
- if (obj_type->meet(TypePtr::NULL_PTR) == obj_type->remove_speculative()) {
-
- // We don't know if it's null or not. Need null-check.
- enum { _not_null_path = 1, _null_path, PATH_LIMIT };
- RegionNode* region = new RegionNode(PATH_LIMIT);
- Node* phi = new PhiNode(region, obj_type);
- Node* null_ctrl = __ top();
- Node* not_null_obj = __ null_check_oop(obj, &null_ctrl);
-
- region->init_req(_null_path, null_ctrl);
- phi ->init_req(_null_path, __ zerocon(T_OBJECT));
-
- Node* ctrl = use_ctrl ? __ control() : NULL;
- ShenandoahReadBarrierNode* rb = new ShenandoahReadBarrierNode(ctrl, mem, not_null_obj, allow_fromspace);
- Node* n = __ gvn().transform(rb);
-
- region->init_req(_not_null_path, __ control());
- phi ->init_req(_not_null_path, n);
-
- __ set_control(__ gvn().transform(region));
- __ record_for_igvn(region);
- return __ gvn().transform(phi);
-
- } else {
- // We know it is not null. Simple barrier is sufficient.
- Node* ctrl = use_ctrl ? __ control() : NULL;
- ShenandoahReadBarrierNode* rb = new ShenandoahReadBarrierNode(ctrl, mem, obj, allow_fromspace);
- Node* n = __ gvn().transform(rb);
- __ record_for_igvn(n);
- return n;
- }
-}
-
-Node* ShenandoahBarrierSetC2::shenandoah_write_barrier_helper(GraphKit* kit, Node* obj, const TypePtr* adr_type) const {
- ShenandoahWriteBarrierNode* wb = new ShenandoahWriteBarrierNode(kit->C, kit->control(), kit->memory(adr_type), obj);
- Node* n = __ gvn().transform(wb);
- if (n == wb) { // New barrier needs memory projection.
- Node* proj = __ gvn().transform(new ShenandoahWBMemProjNode(n));
- __ set_memory(proj, adr_type);
- }
- return n;
-}
-
-Node* ShenandoahBarrierSetC2::shenandoah_write_barrier(GraphKit* kit, Node* obj) const {
- if (ShenandoahWriteBarrier) {
- obj = shenandoah_write_barrier_impl(kit, obj);
- }
- return obj;
-}
-
-Node* ShenandoahBarrierSetC2::shenandoah_write_barrier_impl(GraphKit* kit, Node* obj) const {
- if (! ShenandoahBarrierNode::needs_barrier(&__ gvn(), NULL, obj, NULL, true)) {
- return obj;
- }
- const Type* obj_type = obj->bottom_type();
- const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type);
- Node* n = shenandoah_write_barrier_helper(kit, obj, adr_type);
- __ record_for_igvn(n);
- return n;
-}
+#define __ kit->
bool ShenandoahBarrierSetC2::satb_can_remove_pre_barrier(GraphKit* kit, PhaseTransform* phase, Node* adr,
BasicType bt, uint adr_idx) const {
@@ -304,7 +239,7 @@
Node* gc_state = __ AddP(no_base, tls, __ ConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset())));
Node* ld = __ load(__ ctrl(), gc_state, TypeInt::BYTE, T_BYTE, Compile::AliasIdxRaw);
marking = __ AndI(ld, __ ConI(ShenandoahHeap::MARKING));
- assert(ShenandoahWriteBarrierNode::is_gc_state_load(ld), "Should match the shape");
+ assert(ShenandoahBarrierC2Support::is_gc_state_load(ld), "Should match the shape");
// if (!marking)
__ if_then(marking, BoolTest::ne, zero, unlikely); {
@@ -361,7 +296,7 @@
bool ShenandoahBarrierSetC2::is_shenandoah_wb_call(Node* call) {
return call->is_CallLeaf() &&
- call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT);
+ call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT);
}
bool ShenandoahBarrierSetC2::is_shenandoah_marking_if(PhaseTransform *phase, Node* n) {
@@ -549,88 +484,6 @@
return TypeFunc::make(domain, range);
}
-void ShenandoahBarrierSetC2::resolve_address(C2Access& access) const {
- const TypePtr* adr_type = access.addr().type();
-
- if ((access.decorators() & IN_NATIVE) == 0 && (adr_type->isa_instptr() || adr_type->isa_aryptr())) {
- int off = adr_type->is_ptr()->offset();
- int base_off = adr_type->isa_instptr() ? instanceOopDesc::base_offset_in_bytes() :
- arrayOopDesc::base_offset_in_bytes(adr_type->is_aryptr()->elem()->array_element_basic_type());
- assert(off != Type::OffsetTop, "unexpected offset");
- if (off == Type::OffsetBot || off >= base_off) {
- DecoratorSet decorators = access.decorators();
- bool is_write = (decorators & C2_WRITE_ACCESS) != 0;
- GraphKit* kit = NULL;
- if (access.is_parse_access()) {
- C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
- kit = parse_access.kit();
- }
- Node* adr = access.addr().node();
- assert(adr->is_AddP(), "unexpected address shape");
- Node* base = adr->in(AddPNode::Base);
-
- if (is_write) {
- if (kit != NULL) {
- base = shenandoah_write_barrier(kit, base);
- } else {
- assert(access.is_opt_access(), "either parse or opt access");
- assert((access.decorators() & C2_ARRAY_COPY) != 0, "can be skipped for clone");
- }
- } else {
- if (adr_type->isa_instptr()) {
- Compile* C = access.gvn().C;
- ciField* field = C->alias_type(adr_type)->field();
-
- // Insert read barrier for Shenandoah.
- if (field != NULL &&
- ((ShenandoahOptimizeStaticFinals && field->is_static() && field->is_final()) ||
- (ShenandoahOptimizeInstanceFinals && !field->is_static() && field->is_final()) ||
- (ShenandoahOptimizeStableFinals && field->is_stable()))) {
- // Skip the barrier for special fields
- } else {
- if (kit != NULL) {
- base = shenandoah_read_barrier(kit, base);
- } else {
- assert(access.is_opt_access(), "either parse or opt access");
- assert((access.decorators() & C2_ARRAY_COPY) != 0, "can be skipped for arraycopy");
- }
- }
- } else {
- if (kit != NULL) {
- base = shenandoah_read_barrier(kit, base);
- } else {
- assert(access.is_opt_access(), "either parse or opt access");
- assert((access.decorators() & C2_ARRAY_COPY) != 0, "can be skipped for arraycopy");
- }
- }
- }
- if (base != adr->in(AddPNode::Base)) {
- assert(kit != NULL, "no barrier should have been added");
-
- Node* address = adr->in(AddPNode::Address);
-
- if (address->is_AddP()) {
- assert(address->in(AddPNode::Base) == adr->in(AddPNode::Base), "unexpected address shape");
- assert(!address->in(AddPNode::Address)->is_AddP(), "unexpected address shape");
- assert(address->in(AddPNode::Address) == adr->in(AddPNode::Base), "unexpected address shape");
- address = address->clone();
- address->set_req(AddPNode::Base, base);
- address->set_req(AddPNode::Address, base);
- address = kit->gvn().transform(address);
- } else {
- assert(address == adr->in(AddPNode::Base), "unexpected address shape");
- address = base;
- }
- adr = adr->clone();
- adr->set_req(AddPNode::Base, base);
- adr->set_req(AddPNode::Address, address);
- adr = kit->gvn().transform(adr);
- access.addr().set_node(adr);
- }
- }
- }
-}
-
Node* ShenandoahBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
DecoratorSet decorators = access.decorators();
@@ -662,44 +515,8 @@
PhaseGVN& gvn = opt_access.gvn();
MergeMemNode* mm = opt_access.mem();
- if (ShenandoahStoreValReadBarrier) {
- RegionNode* region = new RegionNode(3);
- const Type* v_t = gvn.type(val.node());
- Node* phi = new PhiNode(region, v_t->isa_oopptr() ? v_t->is_oopptr()->cast_to_nonconst() : v_t);
- Node* cmp = gvn.transform(new CmpPNode(val.node(), gvn.zerocon(T_OBJECT)));
- Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::ne));
- IfNode* iff = new IfNode(opt_access.ctl(), bol, PROB_LIKELY_MAG(3), COUNT_UNKNOWN);
-
- gvn.transform(iff);
- if (gvn.is_IterGVN()) {
- gvn.is_IterGVN()->_worklist.push(iff);
- } else {
- gvn.record_for_igvn(iff);
- }
-
- Node* null_true = gvn.transform(new IfFalseNode(iff));
- Node* null_false = gvn.transform(new IfTrueNode(iff));
- region->init_req(1, null_true);
- region->init_req(2, null_false);
- phi->init_req(1, gvn.zerocon(T_OBJECT));
- Node* cast = new CastPPNode(val.node(), gvn.type(val.node())->join_speculative(TypePtr::NOTNULL));
- cast->set_req(0, null_false);
- cast = gvn.transform(cast);
- Node* rb = gvn.transform(new ShenandoahReadBarrierNode(null_false, gvn.C->immutable_memory(), cast, false));
- phi->init_req(2, rb);
- opt_access.set_ctl(gvn.transform(region));
- val.set_node(gvn.transform(phi));
- }
if (ShenandoahStoreValEnqueueBarrier) {
- const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(gvn.type(val.node()));
- int alias = gvn.C->get_alias_index(adr_type);
- Node* wb = new ShenandoahWriteBarrierNode(gvn.C, opt_access.ctl(), mm->memory_at(alias), val.node());
- Node* wb_transformed = gvn.transform(wb);
- Node* enqueue = gvn.transform(new ShenandoahEnqueueBarrierNode(wb_transformed));
- if (wb_transformed == wb) {
- Node* proj = gvn.transform(new ShenandoahWBMemProjNode(wb));
- mm->set_memory_at(alias, proj);
- }
+ Node* enqueue = gvn.transform(new ShenandoahEnqueueBarrierNode(val.node()));
val.set_node(enqueue);
}
}
@@ -724,6 +541,17 @@
Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : top;
Node* load = BarrierSetC2::load_at_resolved(access, val_type);
+ if (access.is_oop()) {
+ if (ShenandoahLoadRefBarrier) {
+ load = new ShenandoahLoadReferenceBarrierNode(NULL, load);
+ if (access.is_parse_access()) {
+ load = static_cast<C2ParseAccess &>(access).kit()->gvn().transform(load);
+ } else {
+ load = static_cast<C2OptAccess &>(access).gvn().transform(load);
+ }
+ }
+ }
+
// If we are reading the value of the referent field of a Reference
// object (either by using Unsafe directly or through reflection)
// then, if SATB is enabled, we need to record the referent in an
@@ -797,9 +625,10 @@
#ifdef _LP64
if (adr->bottom_type()->is_ptr_to_narrowoop()) {
- return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
+ load_store = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
}
#endif
+ load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, load_store));
return load_store;
}
return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
@@ -867,6 +696,7 @@
}
Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type);
if (access.is_oop()) {
+ result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, result));
shenandoah_write_barrier_pre(kit, false /* do_load */,
NULL, NULL, max_juint, NULL, NULL,
result /* pre_val */, T_OBJECT);
@@ -876,19 +706,9 @@
void ShenandoahBarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const {
assert(!src->is_AddP(), "unexpected input");
- src = shenandoah_read_barrier(kit, src);
BarrierSetC2::clone(kit, src, dst, size, is_array);
}
-Node* ShenandoahBarrierSetC2::resolve(GraphKit* kit, Node* n, DecoratorSet decorators) const {
- bool is_write = decorators & ACCESS_WRITE;
- if (is_write) {
- return shenandoah_write_barrier(kit, n);
- } else {
- return shenandoah_read_barrier(kit, n);
- }
-}
-
Node* ShenandoahBarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* ctrl, Node* mem, Node* toobig_false, Node* size_in_bytes,
Node*& i_o, Node*& needgc_ctrl,
Node*& fast_oop_ctrl, Node*& fast_oop_rawmem,
@@ -915,6 +735,7 @@
// Support for GC barriers emitted during parsing
bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const {
+ if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) return true;
if (node->Opcode() != Op_CallLeaf && node->Opcode() != Op_CallLeafNoFP) {
return false;
}
@@ -929,26 +750,30 @@
}
Node* ShenandoahBarrierSetC2::step_over_gc_barrier(Node* c) const {
- return ShenandoahBarrierNode::skip_through_barrier(c);
+ if (c->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
+ return c->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
+ }
+ if (c->Opcode() == Op_ShenandoahEnqueueBarrier) {
+ c = c->in(1);
+ }
+ return c;
}
bool ShenandoahBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const {
- return !ShenandoahWriteBarrierNode::expand(C, igvn);
+ return !ShenandoahBarrierC2Support::expand(C, igvn);
}
bool ShenandoahBarrierSetC2::optimize_loops(PhaseIdealLoop* phase, LoopOptsMode mode, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const {
if (mode == LoopOptsShenandoahExpand) {
assert(UseShenandoahGC, "only for shenandoah");
- ShenandoahWriteBarrierNode::pin_and_expand(phase);
+ ShenandoahBarrierC2Support::pin_and_expand(phase);
return true;
} else if (mode == LoopOptsShenandoahPostExpand) {
assert(UseShenandoahGC, "only for shenandoah");
visited.Clear();
- ShenandoahWriteBarrierNode::optimize_after_expansion(visited, nstack, worklist, phase);
+ ShenandoahBarrierC2Support::optimize_after_expansion(visited, nstack, worklist, phase);
return true;
}
- GrowableArray<MemoryGraphFixer*> memory_graph_fixers;
- ShenandoahWriteBarrierNode::optimize_before_expansion(phase, memory_graph_fixers, false);
return false;
}
@@ -957,7 +782,6 @@
if (!is_oop) {
return false;
}
-
if (tightly_coupled_alloc) {
if (phase == Optimization) {
return false;
@@ -985,7 +809,7 @@
}
} else {
return true;
- }
+ }
} else if (src_type->isa_aryptr()) {
BasicType src_elem = src_type->klass()->as_array_klass()->element_type()->basic_type();
if (src_elem == T_OBJECT || src_elem == T_ARRAY) {
@@ -1038,14 +862,20 @@
// Support for macro expanded GC barriers
void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
- if (node->Opcode() == Op_ShenandoahWriteBarrier) {
- state()->add_shenandoah_barrier((ShenandoahWriteBarrierNode*) node);
+ if (node->Opcode() == Op_ShenandoahEnqueueBarrier) {
+ state()->add_enqueue_barrier((ShenandoahEnqueueBarrierNode*) node);
+ }
+ if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
+ state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
}
}
void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
- if (node->Opcode() == Op_ShenandoahWriteBarrier) {
- state()->remove_shenandoah_barrier((ShenandoahWriteBarrierNode*) node);
+ if (node->Opcode() == Op_ShenandoahEnqueueBarrier) {
+ state()->remove_enqueue_barrier((ShenandoahEnqueueBarrierNode*) node);
+ }
+ if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
+ state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
}
}
@@ -1091,19 +921,18 @@
}
}
}
- for (int i = state()->shenandoah_barriers_count()-1; i >= 0; i--) {
- ShenandoahWriteBarrierNode* n = state()->shenandoah_barrier(i);
+ for (int i = state()->enqueue_barriers_count() - 1; i >= 0; i--) {
+ ShenandoahEnqueueBarrierNode* n = state()->enqueue_barrier(i);
if (!useful.member(n)) {
- state()->remove_shenandoah_barrier(n);
+ state()->remove_enqueue_barrier(n);
}
}
-
-}
-
-bool ShenandoahBarrierSetC2::has_special_unique_user(const Node* node) const {
- assert(node->outcnt() == 1, "match only for unique out");
- Node* n = node->unique_out();
- return node->Opcode() == Op_ShenandoahWriteBarrier && n->Opcode() == Op_ShenandoahWBMemProj;
+ for (int i = state()->load_reference_barriers_count() - 1; i >= 0; i--) {
+ ShenandoahLoadReferenceBarrierNode* n = state()->load_reference_barrier(i);
+ if (!useful.member(n)) {
+ state()->remove_load_reference_barrier(n);
+ }
+ }
}
void ShenandoahBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {}
@@ -1123,7 +952,7 @@
#ifdef ASSERT
void ShenandoahBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const {
if (ShenandoahVerifyOptoBarriers && phase == BarrierSetC2::BeforeExpand) {
- ShenandoahBarrierNode::verify(Compile::current()->root());
+ ShenandoahBarrierC2Support::verify(Compile::current()->root());
} else if (phase == BarrierSetC2::BeforeCodeGen) {
// Verify G1 pre-barriers
const int marking_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset());
@@ -1229,7 +1058,7 @@
}
} else if (can_reshape &&
n->Opcode() == Op_If &&
- ShenandoahWriteBarrierNode::is_heap_stable_test(n) &&
+ ShenandoahBarrierC2Support::is_heap_stable_test(n) &&
n->in(0) != NULL) {
Node* dom = n->in(0);
Node* prev_dom = n;
@@ -1237,7 +1066,7 @@
int dist = 16;
// Search up the dominator tree for another heap stable test
while (dom->Opcode() != op || // Not same opcode?
- !ShenandoahWriteBarrierNode::is_heap_stable_test(dom) || // Not same input 1?
+ !ShenandoahBarrierC2Support::is_heap_stable_test(dom) || // Not same input 1?
prev_dom->in(0) != dom) { // One path of test does not dominate?
if (dist < 0) return NULL;
@@ -1258,46 +1087,6 @@
return NULL;
}
-Node* ShenandoahBarrierSetC2::identity_node(PhaseGVN* phase, Node* n) const {
- if (n->is_Load()) {
- Node *mem = n->in(MemNode::Memory);
- Node *value = n->as_Load()->can_see_stored_value(mem, phase);
- if (value) {
- PhaseIterGVN *igvn = phase->is_IterGVN();
- if (igvn != NULL &&
- value->is_Phi() &&
- value->req() > 2 &&
- value->in(1) != NULL &&
- value->in(1)->is_ShenandoahBarrier()) {
- if (igvn->_worklist.member(value) ||
- igvn->_worklist.member(value->in(0)) ||
- (value->in(0)->in(1) != NULL &&
- value->in(0)->in(1)->is_IfProj() &&
- (igvn->_worklist.member(value->in(0)->in(1)) ||
- (value->in(0)->in(1)->in(0) != NULL &&
- igvn->_worklist.member(value->in(0)->in(1)->in(0)))))) {
- igvn->_worklist.push(n);
- return n;
- }
- }
- // (This works even when value is a Con, but LoadNode::Value
- // usually runs first, producing the singleton type of the Con.)
- Node *value_no_barrier = step_over_gc_barrier(value->Opcode() == Op_EncodeP ? value->in(1) : value);
- if (value->Opcode() == Op_EncodeP) {
- if (value_no_barrier != value->in(1)) {
- Node *encode = value->clone();
- encode->set_req(1, value_no_barrier);
- encode = phase->transform(encode);
- return encode;
- }
- } else {
- return value_no_barrier;
- }
- }
- }
- return n;
-}
-
bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
Node* u = n->fast_out(i);
@@ -1308,20 +1097,6 @@
return n->outcnt() > 0;
}
-bool ShenandoahBarrierSetC2::flatten_gc_alias_type(const TypePtr*& adr_type) const {
- int offset = adr_type->offset();
- if (offset == ShenandoahBrooksPointer::byte_offset()) {
- if (adr_type->isa_aryptr()) {
- adr_type = TypeAryPtr::make(adr_type->ptr(), adr_type->isa_aryptr()->ary(), adr_type->isa_aryptr()->klass(), false, offset);
- } else if (adr_type->isa_instptr()) {
- adr_type = TypeInstPtr::make(adr_type->ptr(), ciEnv::current()->Object_klass(), false, NULL, offset);
- }
- return true;
- } else {
- return false;
- }
-}
-
bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode) const {
switch (opcode) {
case Op_CallLeaf:
@@ -1356,9 +1131,7 @@
}
#endif
return true;
- case Op_ShenandoahReadBarrier:
- return true;
- case Op_ShenandoahWriteBarrier:
+ case Op_ShenandoahLoadReferenceBarrier:
assert(false, "should have been expanded already");
return true;
default:
@@ -1366,17 +1139,6 @@
}
}
-#ifdef ASSERT
-bool ShenandoahBarrierSetC2::verify_gc_alias_type(const TypePtr* adr_type, int offset) const {
- if (offset == ShenandoahBrooksPointer::byte_offset() &&
- (adr_type->base() == Type::AryPtr || adr_type->base() == Type::OopPtr)) {
- return true;
- } else {
- return false;
- }
-}
-#endif
-
bool ShenandoahBarrierSetC2::escape_add_to_con_graph(ConnectionGraph* conn_graph, PhaseGVN* gvn, Unique_Node_List* delayed_worklist, Node* n, uint opcode) const {
switch (opcode) {
case Op_ShenandoahCompareAndExchangeP:
@@ -1412,15 +1174,12 @@
}
return false;
}
- case Op_ShenandoahReadBarrier:
- case Op_ShenandoahWriteBarrier:
- // Barriers 'pass through' its arguments. I.e. what goes in, comes out.
- // It doesn't escape.
- conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahBarrierNode::ValueIn), delayed_worklist);
- break;
case Op_ShenandoahEnqueueBarrier:
conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(1), delayed_worklist);
break;
+ case Op_ShenandoahLoadReferenceBarrier:
+ conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahLoadReferenceBarrierNode::ValueIn), delayed_worklist);
+ return true;
default:
// Nothing
break;
@@ -1441,15 +1200,12 @@
case Op_ShenandoahWeakCompareAndSwapP:
case Op_ShenandoahWeakCompareAndSwapN:
return conn_graph->add_final_edges_unsafe_access(n, opcode);
- case Op_ShenandoahReadBarrier:
- case Op_ShenandoahWriteBarrier:
- // Barriers 'pass through' its arguments. I.e. what goes in, comes out.
- // It doesn't escape.
- conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahBarrierNode::ValueIn), NULL);
- return true;
case Op_ShenandoahEnqueueBarrier:
conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(1), NULL);
return true;
+ case Op_ShenandoahLoadReferenceBarrier:
+ conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahLoadReferenceBarrierNode::ValueIn), NULL);
+ return true;
default:
// Nothing
break;
@@ -1464,21 +1220,7 @@
}
bool ShenandoahBarrierSetC2::escape_is_barrier_node(Node* n) const {
- return n->is_ShenandoahBarrier();
-}
-
-bool ShenandoahBarrierSetC2::matcher_find_shared_visit(Matcher* matcher, Matcher::MStack& mstack, Node* n, uint opcode, bool& mem_op, int& mem_addr_idx) const {
- switch (opcode) {
- case Op_ShenandoahReadBarrier:
- if (n->in(ShenandoahBarrierNode::ValueIn)->is_DecodeNarrowPtr()) {
- matcher->set_shared(n->in(ShenandoahBarrierNode::ValueIn)->in(1));
- }
- matcher->set_shared(n);
- return true;
- default:
- break;
- }
- return false;
+ return n->Opcode() == Op_ShenandoahLoadReferenceBarrier;
}
bool ShenandoahBarrierSetC2::matcher_find_shared_post_visit(Matcher* matcher, Node* n, uint opcode) const {
@@ -1510,62 +1252,3 @@
xop == Op_ShenandoahCompareAndSwapN ||
xop == Op_ShenandoahCompareAndSwapP;
}
-
-void ShenandoahBarrierSetC2::igvn_add_users_to_worklist(PhaseIterGVN* igvn, Node* use) const {
- if (use->is_ShenandoahBarrier()) {
- for (DUIterator_Fast i2max, i2 = use->fast_outs(i2max); i2 < i2max; i2++) {
- Node* u = use->fast_out(i2);
- Node* cmp = use->find_out_with(Op_CmpP);
- if (u->Opcode() == Op_CmpP) {
- igvn->_worklist.push(cmp);
- }
- }
- }
-}
-
-void ShenandoahBarrierSetC2::ccp_analyze(PhaseCCP* ccp, Unique_Node_List& worklist, Node* use) const {
- if (use->is_ShenandoahBarrier()) {
- for (DUIterator_Fast i2max, i2 = use->fast_outs(i2max); i2 < i2max; i2++) {
- Node* p = use->fast_out(i2);
- if (p->Opcode() == Op_AddP) {
- for (DUIterator_Fast i3max, i3 = p->fast_outs(i3max); i3 < i3max; i3++) {
- Node* q = p->fast_out(i3);
- if (q->is_Load()) {
- if(q->bottom_type() != ccp->type(q)) {
- worklist.push(q);
- }
- }
- }
- }
- }
- }
-}
-
-Node* ShenandoahBarrierSetC2::split_if_pre(PhaseIdealLoop* phase, Node* n) const {
- if (n->Opcode() == Op_ShenandoahReadBarrier) {
- ((ShenandoahReadBarrierNode*)n)->try_move(phase);
- } else if (n->Opcode() == Op_ShenandoahWriteBarrier) {
- return ((ShenandoahWriteBarrierNode*)n)->try_split_thru_phi(phase);
- }
-
- return NULL;
-}
-
-bool ShenandoahBarrierSetC2::build_loop_late_post(PhaseIdealLoop* phase, Node* n) const {
- return ShenandoahBarrierNode::build_loop_late_post(phase, n);
-}
-
-bool ShenandoahBarrierSetC2::sink_node(PhaseIdealLoop* phase, Node* n, Node* x, Node* x_ctrl, Node* n_ctrl) const {
- if (n->is_ShenandoahBarrier()) {
- return x->as_ShenandoahBarrier()->sink_node(phase, x_ctrl, n_ctrl);
- }
- if (n->is_MergeMem()) {
- // PhaseIdealLoop::split_if_with_blocks_post() would:
- // _igvn._worklist.yank(x);
- // which sometimes causes chains of MergeMem which some of
- // shenandoah specific code doesn't support
- phase->register_new_node(x, x_ctrl);
- return true;
- }
- return false;
-}
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp Tue Apr 02 23:00:22 2019 +0200
@@ -30,14 +30,21 @@
class ShenandoahBarrierSetC2State : public ResourceObj {
private:
- GrowableArray<ShenandoahWriteBarrierNode*>* _shenandoah_barriers;
+ GrowableArray<ShenandoahEnqueueBarrierNode*>* _enqueue_barriers;
+ GrowableArray<ShenandoahLoadReferenceBarrierNode*>* _load_reference_barriers;
public:
ShenandoahBarrierSetC2State(Arena* comp_arena);
- int shenandoah_barriers_count() const;
- ShenandoahWriteBarrierNode* shenandoah_barrier(int idx) const;
- void add_shenandoah_barrier(ShenandoahWriteBarrierNode * n);
- void remove_shenandoah_barrier(ShenandoahWriteBarrierNode * n);
+
+ int enqueue_barriers_count() const;
+ ShenandoahEnqueueBarrierNode* enqueue_barrier(int idx) const;
+ void add_enqueue_barrier(ShenandoahEnqueueBarrierNode* n);
+ void remove_enqueue_barrier(ShenandoahEnqueueBarrierNode * n);
+
+ int load_reference_barriers_count() const;
+ ShenandoahLoadReferenceBarrierNode* load_reference_barrier(int idx) const;
+ void add_load_reference_barrier(ShenandoahLoadReferenceBarrierNode* n);
+ void remove_load_reference_barrier(ShenandoahLoadReferenceBarrierNode * n);
};
class ShenandoahBarrierSetC2 : public BarrierSetC2 {
@@ -66,12 +73,7 @@
BasicType bt) const;
Node* shenandoah_enqueue_barrier(GraphKit* kit, Node* val) const;
- Node* shenandoah_read_barrier(GraphKit* kit, Node* obj) const;
Node* shenandoah_storeval_barrier(GraphKit* kit, Node* obj) const;
- Node* shenandoah_write_barrier(GraphKit* kit, Node* obj) const;
- Node* shenandoah_read_barrier_impl(GraphKit* kit, Node* obj, bool use_ctrl, bool use_mem, bool allow_fromspace) const;
- Node* shenandoah_write_barrier_impl(GraphKit* kit, Node* obj) const;
- Node* shenandoah_write_barrier_helper(GraphKit* kit, Node* obj, const TypePtr* adr_type) const;
void insert_pre_barrier(GraphKit* kit, Node* base_oop, Node* offset,
Node* pre_val, bool need_mem_bar) const;
@@ -79,7 +81,6 @@
static bool clone_needs_postbarrier(ArrayCopyNode *ac, PhaseIterGVN& igvn);
protected:
- virtual void resolve_address(C2Access& access) const;
virtual Node* load_at_resolved(C2Access& access, const Type* val_type) const;
virtual Node* store_at_resolved(C2Access& access, C2AccessValue& val) const;
virtual Node* atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
@@ -102,12 +103,11 @@
static const TypeFunc* write_ref_field_pre_entry_Type();
static const TypeFunc* shenandoah_clone_barrier_Type();
static const TypeFunc* shenandoah_write_barrier_Type();
+ virtual bool has_load_barriers() const { return true; }
// This is the entry-point for the backend to perform accesses through the Access API.
virtual void clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const;
- virtual Node* resolve(GraphKit* kit, Node* n, DecoratorSet decorators) const;
-
virtual Node* obj_allocate(PhaseMacroExpand* macro, Node* ctrl, Node* mem, Node* toobig_false, Node* size_in_bytes,
Node*& i_o, Node*& needgc_ctrl,
Node*& fast_oop_ctrl, Node*& fast_oop_rawmem,
@@ -144,13 +144,7 @@
virtual void verify_gc_barriers(Compile* compile, CompilePhase phase) const;
#endif
- virtual bool flatten_gc_alias_type(const TypePtr*& adr_type) const;
-#ifdef ASSERT
- virtual bool verify_gc_alias_type(const TypePtr* adr_type, int offset) const;
-#endif
-
virtual Node* ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const;
- virtual Node* identity_node(PhaseGVN* phase, Node* n) const;
virtual bool final_graph_reshaping(Compile* compile, Node* n, uint opcode) const;
virtual bool escape_add_to_con_graph(ConnectionGraph* conn_graph, PhaseGVN* gvn, Unique_Node_List* delayed_worklist, Node* n, uint opcode) const;
@@ -158,17 +152,8 @@
virtual bool escape_has_out_with_unsafe_object(Node* n) const;
virtual bool escape_is_barrier_node(Node* n) const;
- virtual bool matcher_find_shared_visit(Matcher* matcher, Matcher::MStack& mstack, Node* n, uint opcode, bool& mem_op, int& mem_addr_idx) const;
virtual bool matcher_find_shared_post_visit(Matcher* matcher, Node* n, uint opcode) const;
virtual bool matcher_is_store_load_barrier(Node* x, uint xop) const;
-
- virtual void igvn_add_users_to_worklist(PhaseIterGVN* igvn, Node* use) const;
- virtual void ccp_analyze(PhaseCCP* ccp, Unique_Node_List& worklist, Node* use) const;
-
- virtual bool has_special_unique_user(const Node* node) const;
- virtual Node* split_if_pre(PhaseIdealLoop* phase, Node* n) const;
- virtual bool build_loop_late_post(PhaseIdealLoop* phase, Node* n) const;
- virtual bool sink_node(PhaseIdealLoop* phase, Node* n, Node* x, Node* x_ctrl, Node* n_ctrl) const;
};
#endif // SHARE_GC_SHENANDOAH_C2_SHENANDOAHBARRIERSETC2_HPP
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp Tue Apr 02 23:00:22 2019 +0200
@@ -41,383 +41,28 @@
#include "opto/runtime.hpp"
#include "opto/subnode.hpp"
-Node* ShenandoahBarrierNode::skip_through_barrier(Node* n) {
- if (n == NULL) {
- return NULL;
- }
- if (n->Opcode() == Op_ShenandoahEnqueueBarrier) {
- n = n->in(1);
- }
-
- if (n->is_ShenandoahBarrier()) {
- return n->in(ValueIn);
- } else if (n->is_Phi() &&
- n->req() == 3 &&
- n->in(1) != NULL &&
- n->in(1)->is_ShenandoahBarrier() &&
- n->in(2) != NULL &&
- n->in(2)->bottom_type() == TypePtr::NULL_PTR &&
- n->in(0) != NULL &&
- n->in(0)->in(1) != NULL &&
- n->in(0)->in(1)->is_IfProj() &&
- n->in(0)->in(2) != NULL &&
- n->in(0)->in(2)->is_IfProj() &&
- n->in(0)->in(1)->in(0) != NULL &&
- n->in(0)->in(1)->in(0) == n->in(0)->in(2)->in(0) &&
- n->in(1)->in(ValueIn)->Opcode() == Op_CastPP) {
- Node* iff = n->in(0)->in(1)->in(0);
- Node* res = n->in(1)->in(ValueIn)->in(1);
- if (iff->is_If() &&
- iff->in(1) != NULL &&
- iff->in(1)->is_Bool() &&
- iff->in(1)->as_Bool()->_test._test == BoolTest::ne &&
- iff->in(1)->in(1) != NULL &&
- iff->in(1)->in(1)->Opcode() == Op_CmpP &&
- iff->in(1)->in(1)->in(1) != NULL &&
- iff->in(1)->in(1)->in(1) == res &&
- iff->in(1)->in(1)->in(2) != NULL &&
- iff->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) {
- return res;
- }
- }
- return n;
-}
-
-bool ShenandoahBarrierNode::needs_barrier(PhaseGVN* phase, ShenandoahBarrierNode* orig, Node* n, Node* rb_mem, bool allow_fromspace) {
- Unique_Node_List visited;
- return needs_barrier_impl(phase, orig, n, rb_mem, allow_fromspace, visited);
-}
-
-bool ShenandoahBarrierNode::needs_barrier_impl(PhaseGVN* phase, ShenandoahBarrierNode* orig, Node* n, Node* rb_mem, bool allow_fromspace, Unique_Node_List &visited) {
- if (visited.member(n)) {
- return false; // Been there.
- }
- visited.push(n);
-
- if (n->is_Allocate()) {
- return false;
- }
-
- if (n->is_Call()) {
- return true;
- }
-
- const Type* type = phase->type(n);
- if (type == Type::TOP) {
- return false;
- }
- if (type->make_ptr()->higher_equal(TypePtr::NULL_PTR)) {
- return false;
- }
- if (type->make_oopptr() && type->make_oopptr()->const_oop() != NULL) {
- return false;
- }
-
- if (ShenandoahOptimizeStableFinals) {
- const TypeAryPtr* ary = type->isa_aryptr();
- if (ary && ary->is_stable() && allow_fromspace) {
- return false;
- }
- }
-
- if (n->is_CheckCastPP() || n->is_ConstraintCast() || n->Opcode() == Op_ShenandoahEnqueueBarrier) {
- return needs_barrier_impl(phase, orig, n->in(1), rb_mem, allow_fromspace, visited);
- }
- if (n->is_Parm()) {
- return true;
- }
- if (n->is_Proj()) {
- return needs_barrier_impl(phase, orig, n->in(0), rb_mem, allow_fromspace, visited);
- }
-
- if (n->Opcode() == Op_ShenandoahWBMemProj) {
- return needs_barrier_impl(phase, orig, n->in(ShenandoahWBMemProjNode::WriteBarrier), rb_mem, allow_fromspace, visited);
- }
- if (n->is_Phi()) {
- bool need_barrier = false;
- for (uint i = 1; i < n->req() && ! need_barrier; i++) {
- Node* input = n->in(i);
- if (input == NULL) {
- need_barrier = true; // Phi not complete yet?
- } else if (needs_barrier_impl(phase, orig, input, rb_mem, allow_fromspace, visited)) {
- need_barrier = true;
- }
- }
- return need_barrier;
- }
- if (n->is_CMove()) {
- return needs_barrier_impl(phase, orig, n->in(CMoveNode::IfFalse), rb_mem, allow_fromspace, visited) ||
- needs_barrier_impl(phase, orig, n->in(CMoveNode::IfTrue ), rb_mem, allow_fromspace, visited);
- }
- if (n->Opcode() == Op_CreateEx) {
- return true;
- }
- if (n->Opcode() == Op_ShenandoahWriteBarrier) {
- return false;
- }
- if (n->Opcode() == Op_ShenandoahReadBarrier) {
- if (rb_mem == n->in(Memory)) {
- return false;
- } else {
- return true;
- }
- }
-
- if (n->Opcode() == Op_LoadP ||
- n->Opcode() == Op_LoadN ||
- n->Opcode() == Op_GetAndSetP ||
- n->Opcode() == Op_CompareAndExchangeP ||
- n->Opcode() == Op_ShenandoahCompareAndExchangeP ||
- n->Opcode() == Op_GetAndSetN ||
- n->Opcode() == Op_CompareAndExchangeN ||
- n->Opcode() == Op_ShenandoahCompareAndExchangeN) {
- return true;
- }
- if (n->Opcode() == Op_DecodeN ||
- n->Opcode() == Op_EncodeP) {
- return needs_barrier_impl(phase, orig, n->in(1), rb_mem, allow_fromspace, visited);
- }
-
-#ifdef ASSERT
- tty->print("need barrier on?: "); n->dump();
- ShouldNotReachHere();
-#endif
- return true;
-}
-
-bool ShenandoahReadBarrierNode::dominates_memory_rb_impl(PhaseGVN* phase,
- Node* b1,
- Node* b2,
- Node* current,
- bool linear) {
- ResourceMark rm;
- VectorSet visited(Thread::current()->resource_area());
- Node_Stack phis(0);
-
- for(int i = 0; i < 10; i++) {
- if (current == NULL) {
- return false;
- } else if (visited.test_set(current->_idx) || current->is_top() || current == b1) {
- current = NULL;
- while (phis.is_nonempty() && current == NULL) {
- uint idx = phis.index();
- Node* phi = phis.node();
- if (idx >= phi->req()) {
- phis.pop();
- } else {
- current = phi->in(idx);
- phis.set_index(idx+1);
- }
- }
- if (current == NULL) {
- return true;
- }
- } else if (current == phase->C->immutable_memory()) {
- return false;
- } else if (current->isa_Phi()) {
- if (!linear) {
+bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) {
+ ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
+ if ((state->enqueue_barriers_count() +
+ state->load_reference_barriers_count()) > 0) {
+ bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion;
+ C->clear_major_progress();
+ PhaseIdealLoop ideal_loop(igvn, LoopOptsShenandoahExpand);
+ if (C->failing()) return false;
+ PhaseIdealLoop::verify(igvn);
+ DEBUG_ONLY(verify_raw_mem(C->root());)
+ if (attempt_more_loopopts) {
+ C->set_major_progress();
+ if (!C->optimize_loops(igvn, LoopOptsShenandoahPostExpand)) {
return false;
}
- phis.push(current, 2);
- current = current->in(1);
- } else if (current->Opcode() == Op_ShenandoahWriteBarrier) {
- const Type* in_type = current->bottom_type();
- const Type* this_type = b2->bottom_type();
- if (is_independent(in_type, this_type)) {
- current = current->in(Memory);
- } else {
- return false;
- }
- } else if (current->Opcode() == Op_ShenandoahWBMemProj) {
- current = current->in(ShenandoahWBMemProjNode::WriteBarrier);
- } else if (current->is_Proj()) {
- current = current->in(0);
- } else if (current->is_Call()) {
- return false; // TODO: Maybe improve by looking at the call's memory effects?
- } else if (current->is_MemBar()) {
- return false; // TODO: Do we need to stop at *any* membar?
- } else if (current->is_MergeMem()) {
- const TypePtr* adr_type = brooks_pointer_type(phase->type(b2));
- uint alias_idx = phase->C->get_alias_index(adr_type);
- current = current->as_MergeMem()->memory_at(alias_idx);
- } else {
-#ifdef ASSERT
- current->dump();
-#endif
- ShouldNotReachHere();
- return false;
- }
- }
- return false;
-}
-
-bool ShenandoahReadBarrierNode::is_independent(Node* mem) {
- if (mem->is_Phi() || mem->is_Proj() || mem->is_MergeMem()) {
- return true;
- } else if (mem->Opcode() == Op_ShenandoahWBMemProj) {
- return true;
- } else if (mem->Opcode() == Op_ShenandoahWriteBarrier) {
- const Type* mem_type = mem->bottom_type();
- const Type* this_type = bottom_type();
- if (is_independent(mem_type, this_type)) {
- return true;
- } else {
- return false;
- }
- } else if (mem->is_Call() || mem->is_MemBar()) {
- return false;
- }
-#ifdef ASSERT
- mem->dump();
-#endif
- ShouldNotReachHere();
- return true;
-}
-
-bool ShenandoahReadBarrierNode::dominates_memory_rb(PhaseGVN* phase, Node* b1, Node* b2, bool linear) {
- return dominates_memory_rb_impl(phase, b1->in(Memory), b2, b2->in(Memory), linear);
-}
-
-bool ShenandoahReadBarrierNode::is_independent(const Type* in_type, const Type* this_type) {
- assert(in_type->isa_oopptr(), "expect oop ptr");
- assert(this_type->isa_oopptr(), "expect oop ptr");
-
- ciKlass* in_kls = in_type->is_oopptr()->klass();
- ciKlass* this_kls = this_type->is_oopptr()->klass();
- if (in_kls != NULL && this_kls != NULL &&
- in_kls->is_loaded() && this_kls->is_loaded() &&
- (!in_kls->is_subclass_of(this_kls)) &&
- (!this_kls->is_subclass_of(in_kls))) {
- return true;
- }
- return false;
-}
-
-Node* ShenandoahReadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) {
- if (! can_reshape) {
- return NULL;
- }
-
- if (in(Memory) == phase->C->immutable_memory()) return NULL;
-
- // If memory input is a MergeMem, take the appropriate slice out of it.
- Node* mem_in = in(Memory);
- if (mem_in->isa_MergeMem()) {
- const TypePtr* adr_type = brooks_pointer_type(bottom_type());
- uint alias_idx = phase->C->get_alias_index(adr_type);
- mem_in = mem_in->as_MergeMem()->memory_at(alias_idx);
- set_req(Memory, mem_in);
- return this;
- }
-
- Node* input = in(Memory);
- if (input->Opcode() == Op_ShenandoahWBMemProj) {
- ResourceMark rm;
- VectorSet seen(Thread::current()->resource_area());
- Node* n = in(Memory);
- while (n->Opcode() == Op_ShenandoahWBMemProj &&
- n->in(ShenandoahWBMemProjNode::WriteBarrier) != NULL &&
- n->in(ShenandoahWBMemProjNode::WriteBarrier)->Opcode() == Op_ShenandoahWriteBarrier &&
- n->in(ShenandoahWBMemProjNode::WriteBarrier)->in(Memory) != NULL) {
- if (seen.test_set(n->_idx)) {
- return NULL; // loop
- }
- n = n->in(ShenandoahWBMemProjNode::WriteBarrier)->in(Memory);
- }
-
- Node* wb = input->in(ShenandoahWBMemProjNode::WriteBarrier);
- const Type* in_type = phase->type(wb);
- // is_top() test not sufficient here: we can come here after CCP
- // in a dead branch of the graph that has not yet been removed.
- if (in_type == Type::TOP) return NULL; // Dead path.
- assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "expect write barrier");
- if (is_independent(in_type, _type)) {
- phase->igvn_rehash_node_delayed(wb);
- set_req(Memory, wb->in(Memory));
- if (can_reshape && input->outcnt() == 0) {
- phase->is_IterGVN()->_worklist.push(input);
- }
- return this;
- }
- }
- return NULL;
-}
-
-ShenandoahWriteBarrierNode::ShenandoahWriteBarrierNode(Compile* C, Node* ctrl, Node* mem, Node* obj)
- : ShenandoahBarrierNode(ctrl, mem, obj, false) {
- assert(UseShenandoahGC && ShenandoahWriteBarrier, "should be enabled");
- ShenandoahBarrierSetC2::bsc2()->state()->add_shenandoah_barrier(this);
-}
-
-Node* ShenandoahWriteBarrierNode::Identity(PhaseGVN* phase) {
- assert(in(0) != NULL, "should have control");
- PhaseIterGVN* igvn = phase->is_IterGVN();
- Node* mem_in = in(Memory);
- Node* mem_proj = NULL;
-
- if (igvn != NULL) {
- mem_proj = find_out_with(Op_ShenandoahWBMemProj);
- if (mem_in == mem_proj) {
- return this;
- }
- }
-
- Node* replacement = Identity_impl(phase);
- if (igvn != NULL) {
- if (replacement != NULL && replacement != this && mem_proj != NULL) {
- igvn->replace_node(mem_proj, mem_in);
- }
- }
- return replacement;
-}
-
-Node* ShenandoahWriteBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) {
- assert(in(0) != NULL, "should have control");
- if (!can_reshape) {
- return NULL;
- }
-
- Node* mem_in = in(Memory);
-
- if (mem_in->isa_MergeMem()) {
- const TypePtr* adr_type = brooks_pointer_type(bottom_type());
- uint alias_idx = phase->C->get_alias_index(adr_type);
- mem_in = mem_in->as_MergeMem()->memory_at(alias_idx);
- set_req(Memory, mem_in);
- return this;
- }
-
- Node* val = in(ValueIn);
- if (val->is_ShenandoahBarrier()) {
- set_req(ValueIn, val->in(ValueIn));
- return this;
- }
-
- return NULL;
-}
-
-bool ShenandoahWriteBarrierNode::expand(Compile* C, PhaseIterGVN& igvn) {
- if (UseShenandoahGC) {
- if (ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() > 0 || (!ShenandoahWriteBarrier && ShenandoahStoreValEnqueueBarrier)) {
- bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion;
C->clear_major_progress();
- PhaseIdealLoop ideal_loop(igvn, LoopOptsShenandoahExpand);
- if (C->failing()) return false;
- PhaseIdealLoop::verify(igvn);
- DEBUG_ONLY(ShenandoahBarrierNode::verify_raw_mem(C->root());)
- if (attempt_more_loopopts) {
- C->set_major_progress();
- if (!C->optimize_loops(igvn, LoopOptsShenandoahPostExpand)) {
- return false;
- }
- C->clear_major_progress();
- }
}
}
return true;
}
-bool ShenandoahWriteBarrierNode::is_heap_state_test(Node* iff, int mask) {
+bool ShenandoahBarrierC2Support::is_heap_state_test(Node* iff, int mask) {
if (!UseShenandoahGC) {
return false;
}
@@ -450,11 +95,11 @@
return is_gc_state_load(in1);
}
-bool ShenandoahWriteBarrierNode::is_heap_stable_test(Node* iff) {
+bool ShenandoahBarrierC2Support::is_heap_stable_test(Node* iff) {
return is_heap_state_test(iff, ShenandoahHeap::HAS_FORWARDED);
}
-bool ShenandoahWriteBarrierNode::is_gc_state_load(Node *n) {
+bool ShenandoahBarrierC2Support::is_gc_state_load(Node *n) {
if (!UseShenandoahGC) {
return false;
}
@@ -476,7 +121,7 @@
return true;
}
-bool ShenandoahWriteBarrierNode::has_safepoint_between(Node* start, Node* stop, PhaseIdealLoop *phase) {
+bool ShenandoahBarrierC2Support::has_safepoint_between(Node* start, Node* stop, PhaseIdealLoop *phase) {
assert(phase->is_dominator(stop, start), "bad inputs");
ResourceMark rm;
Unique_Node_List wq;
@@ -500,7 +145,7 @@
return false;
}
-bool ShenandoahWriteBarrierNode::try_common_gc_state_load(Node *n, PhaseIdealLoop *phase) {
+bool ShenandoahBarrierC2Support::try_common_gc_state_load(Node *n, PhaseIdealLoop *phase) {
assert(is_gc_state_load(n), "inconsistent");
Node* addp = n->in(MemNode::Address);
Node* dominator = NULL;
@@ -525,193 +170,8 @@
return true;
}
-bool ShenandoahBarrierNode::dominates_memory_impl(PhaseGVN* phase,
- Node* b1,
- Node* b2,
- Node* current,
- bool linear) {
- ResourceMark rm;
- VectorSet visited(Thread::current()->resource_area());
- Node_Stack phis(0);
-
- for(int i = 0; i < 10; i++) {
- if (current == NULL) {
- return false;
- } else if (visited.test_set(current->_idx) || current->is_top() || current == b1) {
- current = NULL;
- while (phis.is_nonempty() && current == NULL) {
- uint idx = phis.index();
- Node* phi = phis.node();
- if (idx >= phi->req()) {
- phis.pop();
- } else {
- current = phi->in(idx);
- phis.set_index(idx+1);
- }
- }
- if (current == NULL) {
- return true;
- }
- } else if (current == b2) {
- return false;
- } else if (current == phase->C->immutable_memory()) {
- return false;
- } else if (current->isa_Phi()) {
- if (!linear) {
- return false;
- }
- phis.push(current, 2);
- current = current->in(1);
- } else if (current->Opcode() == Op_ShenandoahWriteBarrier) {
- current = current->in(Memory);
- } else if (current->Opcode() == Op_ShenandoahWBMemProj) {
- current = current->in(ShenandoahWBMemProjNode::WriteBarrier);
- } else if (current->is_Proj()) {
- current = current->in(0);
- } else if (current->is_Call()) {
- current = current->in(TypeFunc::Memory);
- } else if (current->is_MemBar()) {
- current = current->in(TypeFunc::Memory);
- } else if (current->is_MergeMem()) {
- const TypePtr* adr_type = brooks_pointer_type(phase->type(b2));
- uint alias_idx = phase->C->get_alias_index(adr_type);
- current = current->as_MergeMem()->memory_at(alias_idx);
- } else {
#ifdef ASSERT
- current->dump();
-#endif
- ShouldNotReachHere();
- return false;
- }
- }
- return false;
-}
-
-/**
- * Determines if b1 dominates b2 through memory inputs. It returns true if:
- * - b1 can be reached by following each branch in b2's memory input (through phis, etc)
- * - or we get back to b2 (i.e. through a loop) without seeing b1
- * In all other cases, (in particular, if we reach immutable_memory without having seen b1)
- * we return false.
- */
-bool ShenandoahBarrierNode::dominates_memory(PhaseGVN* phase, Node* b1, Node* b2, bool linear) {
- return dominates_memory_impl(phase, b1, b2, b2->in(Memory), linear);
-}
-
-Node* ShenandoahBarrierNode::Identity_impl(PhaseGVN* phase) {
- Node* n = in(ValueIn);
-
- Node* rb_mem = Opcode() == Op_ShenandoahReadBarrier ? in(Memory) : NULL;
- if (! needs_barrier(phase, this, n, rb_mem, _allow_fromspace)) {
- return n;
- }
-
- // Try to find a write barrier sibling with identical inputs that we can fold into.
- for (DUIterator i = n->outs(); n->has_out(i); i++) {
- Node* sibling = n->out(i);
- if (sibling == this) {
- continue;
- }
- if (sibling->Opcode() != Op_ShenandoahWriteBarrier) {
- continue;
- }
-
- assert(sibling->in(ValueIn) == in(ValueIn), "sanity");
- assert(sibling->Opcode() == Op_ShenandoahWriteBarrier, "sanity");
-
- if (dominates_memory(phase, sibling, this, phase->is_IterGVN() == NULL)) {
- return sibling;
- }
- }
- return this;
-}
-
-#ifndef PRODUCT
-void ShenandoahBarrierNode::dump_spec(outputStream *st) const {
- const TypePtr* adr = adr_type();
- if (adr == NULL) {
- return;
- }
- st->print(" @");
- adr->dump_on(st);
- st->print(" (");
- Compile::current()->alias_type(adr)->adr_type()->dump_on(st);
- st->print(") ");
-}
-#endif
-
-Node* ShenandoahReadBarrierNode::Identity(PhaseGVN* phase) {
- Node* id = Identity_impl(phase);
-
- if (id == this && phase->is_IterGVN()) {
- Node* n = in(ValueIn);
- // No success in super call. Try to combine identical read barriers.
- for (DUIterator i = n->outs(); n->has_out(i); i++) {
- Node* sibling = n->out(i);
- if (sibling == this || sibling->Opcode() != Op_ShenandoahReadBarrier) {
- continue;
- }
- assert(sibling->in(ValueIn) == in(ValueIn), "sanity");
- if (phase->is_IterGVN()->hash_find(sibling) &&
- sibling->bottom_type() == bottom_type() &&
- sibling->in(Control) == in(Control) &&
- dominates_memory_rb(phase, sibling, this, phase->is_IterGVN() == NULL)) {
- return sibling;
- }
- }
- }
- return id;
-}
-
-const Type* ShenandoahBarrierNode::Value(PhaseGVN* phase) const {
- // Either input is TOP ==> the result is TOP
- const Type *t1 = phase->type(in(Memory));
- if (t1 == Type::TOP) return Type::TOP;
- const Type *t2 = phase->type(in(ValueIn));
- if( t2 == Type::TOP ) return Type::TOP;
-
- if (t2 == TypePtr::NULL_PTR) {
- return _type;
- }
-
- const Type* type = t2->is_oopptr()->cast_to_nonconst();
- return type;
-}
-
-uint ShenandoahBarrierNode::hash() const {
- return TypeNode::hash() + _allow_fromspace;
-}
-
-bool ShenandoahBarrierNode::cmp(const Node& n) const {
- return _allow_fromspace == ((ShenandoahBarrierNode&) n)._allow_fromspace
- && TypeNode::cmp(n);
-}
-
-uint ShenandoahBarrierNode::size_of() const {
- return sizeof(*this);
-}
-
-Node* ShenandoahWBMemProjNode::Identity(PhaseGVN* phase) {
- Node* wb = in(WriteBarrier);
- if (wb->is_top()) return phase->C->top(); // Dead path.
-
- assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "expect write barrier");
- PhaseIterGVN* igvn = phase->is_IterGVN();
- // We can't do the below unless the graph is fully constructed.
- if (igvn == NULL) {
- return this;
- }
-
- // If the mem projection has no barrier users, it's not needed anymore.
- if (wb->outcnt() == 1) {
- return wb->in(ShenandoahBarrierNode::Memory);
- }
-
- return this;
-}
-
-#ifdef ASSERT
-bool ShenandoahBarrierNode::verify_helper(Node* in, Node_Stack& phis, VectorSet& visited, verify_type t, bool trace, Unique_Node_List& barriers_used) {
+bool ShenandoahBarrierC2Support::verify_helper(Node* in, Node_Stack& phis, VectorSet& visited, verify_type t, bool trace, Unique_Node_List& barriers_used) {
assert(phis.size() == 0, "");
while (true) {
@@ -732,12 +192,24 @@
in = in->in(AddPNode::Address);
continue;
} else if (in->is_Con()) {
- if (trace) {tty->print("Found constant"); in->dump();}
- } else if (in->is_ShenandoahBarrier()) {
+ if (trace) {
+ tty->print("Found constant");
+ in->dump();
+ }
+ } else if (in->Opcode() == Op_Parm) {
+ if (trace) {
+ tty->print("Found argument");
+ }
+ } else if (in->Opcode() == Op_CreateEx) {
+ if (trace) {
+ tty->print("Found create-exception");
+ }
+ } else if (in->Opcode() == Op_LoadP && in->adr_type() == TypeRawPtr::BOTTOM) {
+ if (trace) {
+ tty->print("Found raw LoadP (OSR argument?)");
+ }
+ } else if (in->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
if (t == ShenandoahOopStore) {
- if (in->Opcode() != Op_ShenandoahWriteBarrier) {
- return false;
- }
uint i = 0;
for (; i < phis.size(); i++) {
Node* n = phis.node_at(i);
@@ -748,8 +220,6 @@
if (i == phis.size()) {
return false;
}
- } else if (t == ShenandoahStore && in->Opcode() != Op_ShenandoahWriteBarrier) {
- return false;
}
barriers_used.push(in);
if (trace) {tty->print("Found barrier"); in->dump();}
@@ -763,7 +233,14 @@
in = in->in(1);
continue;
} else if (in->is_Proj() && in->in(0)->is_Allocate()) {
- if (trace) {tty->print("Found alloc"); in->in(0)->dump();}
+ if (trace) {
+ tty->print("Found alloc");
+ in->in(0)->dump();
+ }
+ } else if (in->is_Proj() && (in->in(0)->Opcode() == Op_CallStaticJava || in->in(0)->Opcode() == Op_CallDynamicJava)) {
+ if (trace) {
+ tty->print("Found Java call");
+ }
} else if (in->is_Phi()) {
if (!visited.test_set(in->_idx)) {
if (trace) {tty->print("Pushed phi:"); in->dump();}
@@ -809,7 +286,7 @@
return true;
}
-void ShenandoahBarrierNode::report_verify_failure(const char *msg, Node *n1, Node *n2) {
+void ShenandoahBarrierC2Support::report_verify_failure(const char* msg, Node* n1, Node* n2) {
if (n1 != NULL) {
n1->dump(+10);
}
@@ -819,7 +296,7 @@
fatal("%s", msg);
}
-void ShenandoahBarrierNode::verify(RootNode* root) {
+void ShenandoahBarrierC2Support::verify(RootNode* root) {
ResourceMark rm;
Unique_Node_List wq;
GrowableArray<Node*> barriers;
@@ -871,7 +348,7 @@
}
}
- if (verify && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahLoad, trace, barriers_used)) {
+ if (verify && !verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahLoad, trace, barriers_used)) {
report_verify_failure("Shenandoah verification: Load should have barriers", n);
}
}
@@ -899,11 +376,11 @@
}
}
- if (verify && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) {
+ if (verify && !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) {
report_verify_failure("Shenandoah verification: Store should have barriers", n);
}
}
- if (!ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
+ if (!verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
report_verify_failure("Shenandoah verification: Store (address) should have barriers", n);
}
} else if (n->Opcode() == Op_CmpP) {
@@ -926,26 +403,26 @@
} else {
assert(in2->bottom_type()->isa_oopptr(), "");
- if (!ShenandoahBarrierNode::verify_helper(in1, phis, visited, ShenandoahStore, trace, barriers_used) ||
- !ShenandoahBarrierNode::verify_helper(in2, phis, visited, ShenandoahStore, trace, barriers_used)) {
+ if (!verify_helper(in1, phis, visited, ShenandoahStore, trace, barriers_used) ||
+ !verify_helper(in2, phis, visited, ShenandoahStore, trace, barriers_used)) {
report_verify_failure("Shenandoah verification: Cmp should have barriers", n);
}
}
if (verify_no_useless_barrier &&
mark_inputs &&
- (!ShenandoahBarrierNode::verify_helper(in1, phis, visited, ShenandoahValue, trace, barriers_used) ||
- !ShenandoahBarrierNode::verify_helper(in2, phis, visited, ShenandoahValue, trace, barriers_used))) {
+ (!verify_helper(in1, phis, visited, ShenandoahValue, trace, barriers_used) ||
+ !verify_helper(in2, phis, visited, ShenandoahValue, trace, barriers_used))) {
phis.clear();
visited.Reset();
}
}
} else if (n->is_LoadStore()) {
if (n->in(MemNode::ValueIn)->bottom_type()->make_ptr() &&
- !ShenandoahBarrierNode::verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) {
+ !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) {
report_verify_failure("Shenandoah verification: LoadStore (value) should have barriers", n);
}
- if (n->in(MemNode::Address)->bottom_type()->make_oopptr() && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
+ if (n->in(MemNode::Address)->bottom_type()->make_oopptr() && !verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
report_verify_failure("Shenandoah verification: LoadStore (address) should have barriers", n);
}
} else if (n->Opcode() == Op_CallLeafNoFP || n->Opcode() == Op_CallLeaf) {
@@ -1041,13 +518,13 @@
}
}
}
- if (!ShenandoahBarrierNode::verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
- !ShenandoahBarrierNode::verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
+ if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
+ !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
}
} else if (strlen(call->_name) > 5 &&
!strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
- if (!ShenandoahBarrierNode::verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
+ if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
report_verify_failure("Shenandoah verification: _fill should have barriers", n);
}
} else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
@@ -1067,7 +544,7 @@
if (pos == -1) {
break;
}
- if (!ShenandoahBarrierNode::verify_helper(call->in(pos), phis, visited, calls[i].args[j].t, trace, barriers_used)) {
+ if (!verify_helper(call->in(pos), phis, visited, calls[i].args[j].t, trace, barriers_used)) {
report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
}
}
@@ -1090,15 +567,8 @@
}
}
}
- } else if (n->is_ShenandoahBarrier()) {
- assert(!barriers.contains(n), "");
- assert(n->Opcode() != Op_ShenandoahWriteBarrier || n->find_out_with(Op_ShenandoahWBMemProj) != NULL, "bad shenandoah write barrier");
- assert(n->Opcode() != Op_ShenandoahWriteBarrier || n->outcnt() > 1, "bad shenandoah write barrier");
- barriers.push(n);
- } else if (n->Opcode() == Op_ShenandoahEnqueueBarrier) {
+ } else if (n->Opcode() == Op_ShenandoahEnqueueBarrier || n->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
// skip
- } else if (n->Opcode() == Op_ShenandoahWBMemProj) {
- assert(n->in(0) == NULL && n->in(ShenandoahWBMemProjNode::WriteBarrier)->Opcode() == Op_ShenandoahWriteBarrier, "strange ShenandoahWBMemProj");
} else if (n->is_AddP()
|| n->is_Phi()
|| n->is_ConstraintCast()
@@ -1165,7 +635,7 @@
if (pos == -1) {
break;
}
- if (!ShenandoahBarrierNode::verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
+ if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
}
}
@@ -1193,7 +663,7 @@
SafePointNode* sfpt = n->as_SafePoint();
if (verify_no_useless_barrier && sfpt->jvms() != NULL) {
for (uint i = sfpt->jvms()->scloff(); i < sfpt->jvms()->endoff(); i++) {
- if (!ShenandoahBarrierNode::verify_helper(sfpt->in(i), phis, visited, ShenandoahLoad, trace, barriers_used)) {
+ if (!verify_helper(sfpt->in(i), phis, visited, ShenandoahLoad, trace, barriers_used)) {
phis.clear();
visited.Reset();
}
@@ -1227,9 +697,8 @@
n->Opcode() == Op_SCMemProj ||
n->Opcode() == Op_EncodeP ||
n->Opcode() == Op_DecodeN ||
- n->Opcode() == Op_ShenandoahWriteBarrier ||
- n->Opcode() == Op_ShenandoahWBMemProj ||
- n->Opcode() == Op_ShenandoahEnqueueBarrier)) {
+ n->Opcode() == Op_ShenandoahEnqueueBarrier ||
+ n->Opcode() == Op_ShenandoahLoadReferenceBarrier)) {
if (m->bottom_type()->make_oopptr() && m->bottom_type()->make_oopptr()->meet(TypePtr::NULL_PTR) == m->bottom_type()) {
report_verify_failure("Shenandoah verification: null input", n, m);
}
@@ -1251,7 +720,7 @@
}
#endif
-bool ShenandoahBarrierNode::is_dominator_same_ctrl(Node*c, Node* d, Node* n, PhaseIdealLoop* phase) {
+bool ShenandoahBarrierC2Support::is_dominator_same_ctrl(Node* c, Node* d, Node* n, PhaseIdealLoop* phase) {
// That both nodes have the same control is not sufficient to prove
// domination, verify that there's no path from d to n
ResourceMark rm;
@@ -1275,7 +744,7 @@
return true;
}
-bool ShenandoahBarrierNode::is_dominator(Node *d_c, Node *n_c, Node* d, Node* n, PhaseIdealLoop* phase) {
+bool ShenandoahBarrierC2Support::is_dominator(Node* d_c, Node* n_c, Node* d, Node* n, PhaseIdealLoop* phase) {
if (d_c != n_c) {
return phase->is_dominator(d_c, n_c);
}
@@ -1290,15 +759,11 @@
res = mem->in(TypeFunc::Memory);
} else if (mem->is_Phi()) {
res = mem->in(1);
- } else if (mem->is_ShenandoahBarrier()) {
- res = mem->in(ShenandoahBarrierNode::Memory);
} else if (mem->is_MergeMem()) {
res = mem->as_MergeMem()->memory_at(alias);
} else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) {
assert(alias = Compile::AliasIdxRaw, "following raw memory can't lead to a barrier");
res = mem->in(MemNode::Memory);
- } else if (mem->Opcode() == Op_ShenandoahWBMemProj) {
- res = mem->in(ShenandoahWBMemProjNode::WriteBarrier);
} else {
#ifdef ASSERT
mem->dump();
@@ -1308,7 +773,7 @@
return res;
}
-Node* ShenandoahBarrierNode::no_branches(Node* c, Node* dom, bool allow_one_proj, PhaseIdealLoop* phase) {
+Node* ShenandoahBarrierC2Support::no_branches(Node* c, Node* dom, bool allow_one_proj, PhaseIdealLoop* phase) {
Node* iffproj = NULL;
while (c != dom) {
Node* next = phase->idom(c);
@@ -1373,270 +838,7 @@
return iffproj;
}
-bool ShenandoahBarrierNode::build_loop_late_post(PhaseIdealLoop* phase, Node* n) {
- if (n->Opcode() == Op_ShenandoahReadBarrier ||
- n->Opcode() == Op_ShenandoahWriteBarrier ||
- n->Opcode() == Op_ShenandoahWBMemProj) {
-
- phase->build_loop_late_post_work(n, false);
-
- if (n->Opcode() == Op_ShenandoahWriteBarrier) {
- // The write barrier and its memory proj must have the same
- // control otherwise some loop opts could put nodes (Phis) between
- // them
- Node* proj = n->find_out_with(Op_ShenandoahWBMemProj);
- if (proj != NULL) {
- phase->set_ctrl_and_loop(proj, phase->get_ctrl(n));
- }
- }
- return true;
- }
- return false;
-}
-
-bool ShenandoahBarrierNode::sink_node(PhaseIdealLoop* phase, Node* ctrl, Node* n_ctrl) {
- ctrl = phase->find_non_split_ctrl(ctrl);
- assert(phase->dom_depth(n_ctrl) <= phase->dom_depth(ctrl), "n is later than its clone");
- set_req(0, ctrl);
- phase->register_new_node(this, ctrl);
- return true;
-}
-
-#ifdef ASSERT
-void ShenandoahWriteBarrierNode::memory_dominates_all_paths_helper(Node* c, Node* rep_ctrl, Unique_Node_List& controls, PhaseIdealLoop* phase) {
- const bool trace = false;
- if (trace) { tty->print("X control is"); c->dump(); }
-
- uint start = controls.size();
- controls.push(c);
- for (uint i = start; i < controls.size(); i++) {
- Node *n = controls.at(i);
-
- if (trace) { tty->print("X from"); n->dump(); }
-
- if (n == rep_ctrl) {
- continue;
- }
-
- if (n->is_Proj()) {
- Node* n_dom = n->in(0);
- IdealLoopTree* n_dom_loop = phase->get_loop(n_dom);
- if (n->is_IfProj() && n_dom->outcnt() == 2) {
- n_dom_loop = phase->get_loop(n_dom->as_If()->proj_out(n->as_Proj()->_con == 0 ? 1 : 0));
- }
- if (n_dom_loop != phase->ltree_root()) {
- Node* tail = n_dom_loop->tail();
- if (tail->is_Region()) {
- for (uint j = 1; j < tail->req(); j++) {
- if (phase->is_dominator(n_dom, tail->in(j)) && !phase->is_dominator(n, tail->in(j))) {
- assert(phase->is_dominator(rep_ctrl, tail->in(j)), "why are we here?");
- // entering loop from below, mark backedge
- if (trace) { tty->print("X pushing backedge"); tail->in(j)->dump(); }
- controls.push(tail->in(j));
- //assert(n->in(0) == n_dom, "strange flow control");
- }
- }
- } else if (phase->get_loop(n) != n_dom_loop && phase->is_dominator(n_dom, tail)) {
- // entering loop from below, mark backedge
- if (trace) { tty->print("X pushing backedge"); tail->dump(); }
- controls.push(tail);
- //assert(n->in(0) == n_dom, "strange flow control");
- }
- }
- }
-
- if (n->is_Loop()) {
- Node* c = n->in(LoopNode::EntryControl);
- if (trace) { tty->print("X pushing"); c->dump(); }
- controls.push(c);
- } else if (n->is_Region()) {
- for (uint i = 1; i < n->req(); i++) {
- Node* c = n->in(i);
- if (trace) { tty->print("X pushing"); c->dump(); }
- controls.push(c);
- }
- } else {
- Node* c = n->in(0);
- if (trace) { tty->print("X pushing"); c->dump(); }
- controls.push(c);
- }
- }
-}
-
-bool ShenandoahWriteBarrierNode::memory_dominates_all_paths(Node* mem, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) {
- const bool trace = false;
- if (trace) {
- tty->print("XXX mem is"); mem->dump();
- tty->print("XXX rep ctrl is"); rep_ctrl->dump();
- tty->print_cr("XXX alias is %d", alias);
- }
- ResourceMark rm;
- Unique_Node_List wq;
- Unique_Node_List controls;
- wq.push(mem);
- for (uint next = 0; next < wq.size(); next++) {
- Node *nn = wq.at(next);
- if (trace) { tty->print("XX from mem"); nn->dump(); }
- assert(nn->bottom_type() == Type::MEMORY, "memory only");
-
- if (nn->is_Phi()) {
- Node* r = nn->in(0);
- for (DUIterator_Fast jmax, j = r->fast_outs(jmax); j < jmax; j++) {
- Node* u = r->fast_out(j);
- if (u->is_Phi() && u->bottom_type() == Type::MEMORY && u != nn &&
- (u->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(u->adr_type()) == alias)) {
- if (trace) { tty->print("XX Next mem (other phi)"); u->dump(); }
- wq.push(u);
- }
- }
- }
-
- for (DUIterator_Fast imax, i = nn->fast_outs(imax); i < imax; i++) {
- Node* use = nn->fast_out(i);
-
- if (trace) { tty->print("XX use %p", use->adr_type()); use->dump(); }
- if (use->is_CFG() && use->in(TypeFunc::Memory) == nn) {
- Node* c = use->in(0);
- if (phase->is_dominator(rep_ctrl, c)) {
- memory_dominates_all_paths_helper(c, rep_ctrl, controls, phase);
- } else if (use->is_CallStaticJava() && use->as_CallStaticJava()->uncommon_trap_request() != 0 && c->is_Region()) {
- Node* region = c;
- if (trace) { tty->print("XX unc region"); region->dump(); }
- for (uint j = 1; j < region->req(); j++) {
- if (phase->is_dominator(rep_ctrl, region->in(j))) {
- if (trace) { tty->print("XX unc follows"); region->in(j)->dump(); }
- memory_dominates_all_paths_helper(region->in(j), rep_ctrl, controls, phase);
- }
- }
- }
- //continue;
- } else if (use->is_Phi()) {
- assert(use->bottom_type() == Type::MEMORY, "bad phi");
- if ((use->adr_type() == TypePtr::BOTTOM) ||
- phase->C->get_alias_index(use->adr_type()) == alias) {
- for (uint j = 1; j < use->req(); j++) {
- if (use->in(j) == nn) {
- Node* c = use->in(0)->in(j);
- if (phase->is_dominator(rep_ctrl, c)) {
- memory_dominates_all_paths_helper(c, rep_ctrl, controls, phase);
- }
- }
- }
- }
- // continue;
- }
-
- if (use->is_MergeMem()) {
- if (use->as_MergeMem()->memory_at(alias) == nn) {
- if (trace) { tty->print("XX Next mem"); use->dump(); }
- // follow the memory edges
- wq.push(use);
- }
- } else if (use->is_Phi()) {
- assert(use->bottom_type() == Type::MEMORY, "bad phi");
- if ((use->adr_type() == TypePtr::BOTTOM) ||
- phase->C->get_alias_index(use->adr_type()) == alias) {
- if (trace) { tty->print("XX Next mem"); use->dump(); }
- // follow the memory edges
- wq.push(use);
- }
- } else if (use->bottom_type() == Type::MEMORY &&
- (use->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(use->adr_type()) == alias)) {
- if (trace) { tty->print("XX Next mem"); use->dump(); }
- // follow the memory edges
- wq.push(use);
- } else if ((use->is_SafePoint() || use->is_MemBar()) &&
- (use->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(use->adr_type()) == alias)) {
- for (DUIterator_Fast jmax, j = use->fast_outs(jmax); j < jmax; j++) {
- Node* u = use->fast_out(j);
- if (u->bottom_type() == Type::MEMORY) {
- if (trace) { tty->print("XX Next mem"); u->dump(); }
- // follow the memory edges
- wq.push(u);
- }
- }
- } else if (use->Opcode() == Op_ShenandoahWriteBarrier && phase->C->get_alias_index(use->adr_type()) == alias) {
- Node* m = use->find_out_with(Op_ShenandoahWBMemProj);
- if (m != NULL) {
- if (trace) { tty->print("XX Next mem"); m->dump(); }
- // follow the memory edges
- wq.push(m);
- }
- }
- }
- }
-
- if (controls.size() == 0) {
- return false;
- }
-
- for (uint i = 0; i < controls.size(); i++) {
- Node *n = controls.at(i);
-
- if (trace) { tty->print("X checking"); n->dump(); }
-
- if (n->unique_ctrl_out() != NULL) {
- continue;
- }
-
- if (n->Opcode() == Op_NeverBranch) {
- Node* taken = n->as_Multi()->proj_out(0);
- if (!controls.member(taken)) {
- if (trace) { tty->print("X not seen"); taken->dump(); }
- return false;
- }
- continue;
- }
-
- for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
- Node* u = n->fast_out(j);
-
- if (u->is_CFG()) {
- if (!controls.member(u)) {
- if (u->is_Proj() && u->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) {
- if (trace) { tty->print("X not seen but unc"); u->dump(); }
- } else {
- Node* c = u;
- do {
- c = c->unique_ctrl_out();
- } while (c != NULL && c->is_Region());
- if (c != NULL && c->Opcode() == Op_Halt) {
- if (trace) { tty->print("X not seen but halt"); c->dump(); }
- } else {
- if (trace) { tty->print("X not seen"); u->dump(); }
- return false;
- }
- }
- } else {
- if (trace) { tty->print("X seen"); u->dump(); }
- }
- }
- }
- }
- return true;
-}
-#endif
-
-Node* ShenandoahBarrierNode::dom_mem(Node* mem, Node*& mem_ctrl, Node* n, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) {
- ResourceMark rm;
- VectorSet wq(Thread::current()->resource_area());
- wq.set(mem->_idx);
- mem_ctrl = phase->get_ctrl(mem);
- while (!is_dominator(mem_ctrl, rep_ctrl, mem, n, phase)) {
- mem = next_mem(mem, alias);
- if (wq.test_set(mem->_idx)) {
- return NULL; // hit an unexpected loop
- }
- mem_ctrl = phase->ctrl_or_self(mem);
- }
- if (mem->is_MergeMem()) {
- mem = mem->as_MergeMem()->memory_at(alias);
- mem_ctrl = phase->ctrl_or_self(mem);
- }
- return mem;
-}
-
-Node* ShenandoahBarrierNode::dom_mem(Node* mem, Node* ctrl, int alias, Node*& mem_ctrl, PhaseIdealLoop* phase) {
+Node* ShenandoahBarrierC2Support::dom_mem(Node* mem, Node* ctrl, int alias, Node*& mem_ctrl, PhaseIdealLoop* phase) {
ResourceMark rm;
VectorSet wq(Thread::current()->resource_area());
wq.set(mem->_idx);
@@ -1655,650 +857,7 @@
return mem;
}
-static void disconnect_barrier_mem(Node* wb, PhaseIterGVN& igvn) {
- Node* mem_in = wb->in(ShenandoahBarrierNode::Memory);
- Node* proj = wb->find_out_with(Op_ShenandoahWBMemProj);
-
- for (DUIterator_Last imin, i = proj->last_outs(imin); i >= imin; ) {
- Node* u = proj->last_out(i);
- igvn.rehash_node_delayed(u);
- int nb = u->replace_edge(proj, mem_in);
- assert(nb > 0, "no replacement?");
- i -= nb;
- }
-}
-
-Node* ShenandoahWriteBarrierNode::move_above_predicates(LoopNode* cl, Node* val_ctrl, PhaseIdealLoop* phase) {
- Node* entry = cl->skip_strip_mined(-1)->in(LoopNode::EntryControl);
- Node* above_pred = phase->skip_all_loop_predicates(entry);
- Node* ctrl = entry;
- while (ctrl != above_pred) {
- Node* next = ctrl->in(0);
- if (!phase->is_dominator(val_ctrl, next)) {
- break;
- }
- ctrl = next;
- }
- return ctrl;
-}
-
-static MemoryGraphFixer* find_fixer(GrowableArray<MemoryGraphFixer*>& memory_graph_fixers, int alias) {
- for (int i = 0; i < memory_graph_fixers.length(); i++) {
- if (memory_graph_fixers.at(i)->alias() == alias) {
- return memory_graph_fixers.at(i);
- }
- }
- return NULL;
-}
-
-static MemoryGraphFixer* create_fixer(GrowableArray<MemoryGraphFixer*>& memory_graph_fixers, int alias, PhaseIdealLoop* phase, bool include_lsm) {
- assert(find_fixer(memory_graph_fixers, alias) == NULL, "none should exist yet");
- MemoryGraphFixer* fixer = new MemoryGraphFixer(alias, include_lsm, phase);
- memory_graph_fixers.push(fixer);
- return fixer;
-}
-
-void ShenandoahWriteBarrierNode::try_move_before_loop_helper(LoopNode* cl, Node* val_ctrl, GrowableArray<MemoryGraphFixer*>& memory_graph_fixers, PhaseIdealLoop* phase, bool include_lsm, Unique_Node_List& uses) {
- assert(cl->is_Loop(), "bad control");
- Node* ctrl = move_above_predicates(cl, val_ctrl, phase);
- Node* mem_ctrl = NULL;
- int alias = phase->C->get_alias_index(adr_type());
-
- MemoryGraphFixer* fixer = find_fixer(memory_graph_fixers, alias);
- if (fixer == NULL) {
- fixer = create_fixer(memory_graph_fixers, alias, phase, include_lsm);
- }
-
- Node* proj = find_out_with(Op_ShenandoahWBMemProj);
-
- fixer->remove(proj);
- Node* mem = fixer->find_mem(ctrl, NULL);
-
- assert(!ShenandoahVerifyOptoBarriers || memory_dominates_all_paths(mem, ctrl, alias, phase), "can't fix the memory graph");
-
- phase->set_ctrl_and_loop(this, ctrl);
- phase->igvn().replace_input_of(this, Control, ctrl);
-
- disconnect_barrier_mem(this, phase->igvn());
-
- phase->igvn().replace_input_of(this, Memory, mem);
- phase->set_ctrl_and_loop(proj, ctrl);
-
- fixer->fix_mem(ctrl, ctrl, mem, mem, proj, uses);
- assert(proj->outcnt() > 0, "disconnected write barrier");
-}
-
-LoopNode* ShenandoahWriteBarrierNode::try_move_before_pre_loop(Node* c, Node* val_ctrl, PhaseIdealLoop* phase) {
- // A write barrier between a pre and main loop can get in the way of
- // vectorization. Move it above the pre loop if possible
- CountedLoopNode* cl = NULL;
- if (c->is_IfFalse() &&
- c->in(0)->is_CountedLoopEnd()) {
- cl = c->in(0)->as_CountedLoopEnd()->loopnode();
- } else if (c->is_IfProj() &&
- c->in(0)->is_If() &&
- c->in(0)->in(0)->is_IfFalse() &&
- c->in(0)->in(0)->in(0)->is_CountedLoopEnd()) {
- cl = c->in(0)->in(0)->in(0)->as_CountedLoopEnd()->loopnode();
- }
- if (cl != NULL &&
- cl->is_pre_loop() &&
- val_ctrl != cl &&
- phase->is_dominator(val_ctrl, cl)) {
- return cl;
- }
- return NULL;
-}
-
-void ShenandoahWriteBarrierNode::try_move_before_loop(GrowableArray<MemoryGraphFixer*>& memory_graph_fixers, PhaseIdealLoop* phase, bool include_lsm, Unique_Node_List& uses) {
- Node *n_ctrl = phase->get_ctrl(this);
- IdealLoopTree *n_loop = phase->get_loop(n_ctrl);
- Node* val = in(ValueIn);
- Node* val_ctrl = phase->get_ctrl(val);
- if (n_loop != phase->ltree_root() && !n_loop->_irreducible) {
- IdealLoopTree *val_loop = phase->get_loop(val_ctrl);
- Node* mem = in(Memory);
- IdealLoopTree *mem_loop = phase->get_loop(phase->get_ctrl(mem));
- if (!n_loop->is_member(val_loop) &&
- n_loop->is_member(mem_loop)) {
- Node* n_loop_head = n_loop->_head;
-
- if (n_loop_head->is_Loop()) {
- LoopNode* loop = n_loop_head->as_Loop();
- if (n_loop_head->is_CountedLoop() && n_loop_head->as_CountedLoop()->is_main_loop()) {
- LoopNode* res = try_move_before_pre_loop(n_loop_head->in(LoopNode::EntryControl), val_ctrl, phase);
- if (res != NULL) {
- loop = res;
- }
- }
-
- try_move_before_loop_helper(loop, val_ctrl, memory_graph_fixers, phase, include_lsm, uses);
- }
- }
- }
- LoopNode* ctrl = try_move_before_pre_loop(in(0), val_ctrl, phase);
- if (ctrl != NULL) {
- try_move_before_loop_helper(ctrl, val_ctrl, memory_graph_fixers, phase, include_lsm, uses);
- }
-}
-
-Node* ShenandoahWriteBarrierNode::would_subsume(ShenandoahBarrierNode* other, PhaseIdealLoop* phase) {
- Node* val = in(ValueIn);
- Node* val_ctrl = phase->get_ctrl(val);
- Node* other_mem = other->in(Memory);
- Node* other_ctrl = phase->get_ctrl(other);
- Node* this_ctrl = phase->get_ctrl(this);
- IdealLoopTree* this_loop = phase->get_loop(this_ctrl);
- IdealLoopTree* other_loop = phase->get_loop(other_ctrl);
-
- Node* ctrl = phase->dom_lca(other_ctrl, this_ctrl);
-
- if (ctrl->is_Proj() &&
- ctrl->in(0)->is_Call() &&
- ctrl->unique_ctrl_out() != NULL &&
- ctrl->unique_ctrl_out()->Opcode() == Op_Catch &&
- !phase->is_dominator(val_ctrl, ctrl->in(0)->in(0))) {
- return NULL;
- }
-
- IdealLoopTree* loop = phase->get_loop(ctrl);
-
- // We don't want to move a write barrier in a loop
- // If the LCA is in a inner loop, try a control out of loop if possible
- while (!loop->is_member(this_loop) && (other->Opcode() != Op_ShenandoahWriteBarrier || !loop->is_member(other_loop))) {
- ctrl = phase->idom(ctrl);
- if (ctrl->is_MultiBranch()) {
- ctrl = ctrl->in(0);
- }
- if (ctrl != val_ctrl && phase->is_dominator(ctrl, val_ctrl)) {
- return NULL;
- }
- loop = phase->get_loop(ctrl);
- }
-
- if (ShenandoahDontIncreaseWBFreq) {
- Node* this_iffproj = no_branches(this_ctrl, ctrl, true, phase);
- if (other->Opcode() == Op_ShenandoahWriteBarrier) {
- Node* other_iffproj = no_branches(other_ctrl, ctrl, true, phase);
- if (other_iffproj == NULL || this_iffproj == NULL) {
- return ctrl;
- } else if (other_iffproj != NodeSentinel && this_iffproj != NodeSentinel &&
- other_iffproj->in(0) == this_iffproj->in(0)) {
- return ctrl;
- }
- } else if (this_iffproj == NULL) {
- return ctrl;
- }
- return NULL;
- }
-
- return ctrl;
-}
-
-void ShenandoahWriteBarrierNode::optimize_before_expansion(PhaseIdealLoop* phase, GrowableArray<MemoryGraphFixer*> memory_graph_fixers, bool include_lsm) {
- bool progress = false;
- Unique_Node_List uses;
- do {
- progress = false;
- for (int i = 0; i < ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i++) {
- ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i);
-
- wb->try_move_before_loop(memory_graph_fixers, phase, include_lsm, uses);
-
- Node* val = wb->in(ValueIn);
-
- for (DUIterator_Fast jmax, j = val->fast_outs(jmax); j < jmax; j++) {
- Node* u = val->fast_out(j);
- if (u != wb && u->is_ShenandoahBarrier()) {
- Node* rep_ctrl = wb->would_subsume(u->as_ShenandoahBarrier(), phase);
-
- if (rep_ctrl != NULL) {
- Node* other = u;
- Node* val_ctrl = phase->get_ctrl(val);
- if (rep_ctrl->is_Proj() &&
- rep_ctrl->in(0)->is_Call() &&
- rep_ctrl->unique_ctrl_out() != NULL &&
- rep_ctrl->unique_ctrl_out()->Opcode() == Op_Catch) {
- rep_ctrl = rep_ctrl->in(0)->in(0);
-
- assert(phase->is_dominator(val_ctrl, rep_ctrl), "bad control");
- } else {
- LoopNode* c = ShenandoahWriteBarrierNode::try_move_before_pre_loop(rep_ctrl, val_ctrl, phase);
- if (c != NULL) {
- rep_ctrl = ShenandoahWriteBarrierNode::move_above_predicates(c, val_ctrl, phase);
- } else {
- while (rep_ctrl->is_IfProj()) {
- CallStaticJavaNode* unc = rep_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
- if (unc != NULL) {
- int req = unc->uncommon_trap_request();
- Deoptimization::DeoptReason trap_reason = Deoptimization::trap_request_reason(req);
- if ((trap_reason == Deoptimization::Reason_loop_limit_check ||
- trap_reason == Deoptimization::Reason_predicate ||
- trap_reason == Deoptimization::Reason_profile_predicate) &&
- phase->is_dominator(val_ctrl, rep_ctrl->in(0)->in(0))) {
- rep_ctrl = rep_ctrl->in(0)->in(0);
- continue;
- }
- }
- break;
- }
- }
- }
-
- Node* wb_ctrl = phase->get_ctrl(wb);
- Node* other_ctrl = phase->get_ctrl(other);
- int alias = phase->C->get_alias_index(wb->adr_type());
- MemoryGraphFixer* fixer = find_fixer(memory_graph_fixers, alias);;
- if (!is_dominator(wb_ctrl, other_ctrl, wb, other, phase)) {
- if (fixer == NULL) {
- fixer = create_fixer(memory_graph_fixers, alias, phase, include_lsm);
- }
- Node* mem = fixer->find_mem(rep_ctrl, phase->get_ctrl(other) == rep_ctrl ? other : NULL);
-
- if (mem->has_out_with(Op_Lock) || mem->has_out_with(Op_Unlock)) {
- continue;
- }
-
- Node* wb_proj = wb->find_out_with(Op_ShenandoahWBMemProj);
- fixer->remove(wb_proj);
- Node* mem_for_ctrl = fixer->find_mem(rep_ctrl, NULL);
-
- if (wb->in(Memory) != mem) {
- disconnect_barrier_mem(wb, phase->igvn());
- phase->igvn().replace_input_of(wb, Memory, mem);
- }
- if (rep_ctrl != wb_ctrl) {
- phase->set_ctrl_and_loop(wb, rep_ctrl);
- phase->igvn().replace_input_of(wb, Control, rep_ctrl);
- phase->set_ctrl_and_loop(wb_proj, rep_ctrl);
- progress = true;
- }
-
- fixer->fix_mem(rep_ctrl, rep_ctrl, mem, mem_for_ctrl, wb_proj, uses);
-
- assert(!ShenandoahVerifyOptoBarriers || ShenandoahWriteBarrierNode::memory_dominates_all_paths(mem, rep_ctrl, alias, phase), "can't fix the memory graph");
- }
-
- if (other->Opcode() == Op_ShenandoahWriteBarrier) {
- Node* other_proj = other->find_out_with(Op_ShenandoahWBMemProj);
- if (fixer != NULL) {
- fixer->remove(other_proj);
- }
- phase->igvn().replace_node(other_proj, other->in(Memory));
- }
- phase->igvn().replace_node(other, wb);
- --j; --jmax;
- }
- }
- }
- }
- } while(progress);
-}
-
-// Some code duplication with PhaseIdealLoop::split_if_with_blocks_pre()
-Node* ShenandoahWriteBarrierNode::try_split_thru_phi(PhaseIdealLoop* phase) {
- Node *ctrl = phase->get_ctrl(this);
- if (ctrl == NULL) {
- return this;
- }
- Node *blk = phase->has_local_phi_input(this);
- if (blk == NULL) {
- return this;
- }
-
- if (in(0) != blk) {
- return this;
- }
-
- int policy = blk->req() >> 2;
-
- if (blk->is_CountedLoop()) {
- IdealLoopTree *lp = phase->get_loop(blk);
- if (lp && lp->_rce_candidate) {
- return this;
- }
- }
-
- if (phase->C->live_nodes() > 35000) {
- return this;
- }
-
- uint unique = phase->C->unique();
- Node *phi = phase->split_thru_phi(this, blk, policy);
- if (phi == NULL) {
- return this;
- }
-
- Node* mem_phi = new PhiNode(blk, Type::MEMORY, phase->C->alias_type(adr_type())->adr_type());
- for (uint i = 1; i < blk->req(); i++) {
- Node* n = phi->in(i);
- if (n->Opcode() == Op_ShenandoahWriteBarrier &&
- n->_idx >= unique) {
- Node* proj = new ShenandoahWBMemProjNode(n);
- phase->register_new_node(proj, phase->get_ctrl(n));
- mem_phi->init_req(i, proj);
- } else {
- Node* mem = in(ShenandoahBarrierNode::Memory);
- if (mem->is_Phi() && mem->in(0) == blk) {
- mem = mem->in(i);
- }
- mem_phi->init_req(i, mem);
- }
- }
- phase->register_new_node(mem_phi, blk);
-
-
- Node* proj = find_out_with(Op_ShenandoahWBMemProj);
- phase->igvn().replace_node(proj, mem_phi);
- phase->igvn().replace_node(this, phi);
-
- return phi;
-}
-
-void ShenandoahReadBarrierNode::try_move(PhaseIdealLoop* phase) {
- Node *n_ctrl = phase->get_ctrl(this);
- if (n_ctrl == NULL) {
- return;
- }
- Node* mem = in(MemNode::Memory);
- int alias = phase->C->get_alias_index(adr_type());
- const bool trace = false;
-
-#ifdef ASSERT
- if (trace) { tty->print("Trying to move mem of"); dump(); }
-#endif
-
- Node* new_mem = mem;
-
- ResourceMark rm;
- VectorSet seen(Thread::current()->resource_area());
- Node_List phis;
-
- for (;;) {
-#ifdef ASSERT
- if (trace) { tty->print("Looking for dominator from"); mem->dump(); }
-#endif
- if (mem->is_Proj() && mem->in(0)->is_Start()) {
- if (new_mem != in(MemNode::Memory)) {
-#ifdef ASSERT
- if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); }
-#endif
- phase->igvn().replace_input_of(this, MemNode::Memory, new_mem);
- }
- return;
- }
-
- Node* candidate = mem;
- do {
- if (!is_independent(mem)) {
- if (trace) { tty->print_cr("Not independent"); }
- if (new_mem != in(MemNode::Memory)) {
-#ifdef ASSERT
- if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); }
-#endif
- phase->igvn().replace_input_of(this, MemNode::Memory, new_mem);
- }
- return;
- }
- if (seen.test_set(mem->_idx)) {
- if (trace) { tty->print_cr("Already seen"); }
- ShouldNotReachHere();
- // Strange graph
- if (new_mem != in(MemNode::Memory)) {
-#ifdef ASSERT
- if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); }
-#endif
- phase->igvn().replace_input_of(this, MemNode::Memory, new_mem);
- }
- return;
- }
- if (mem->is_Phi()) {
- phis.push(mem);
- }
- mem = next_mem(mem, alias);
- if (mem->bottom_type() == Type::MEMORY) {
- candidate = mem;
- }
- assert(is_dominator(phase->ctrl_or_self(mem), n_ctrl, mem, this, phase) == phase->is_dominator(phase->ctrl_or_self(mem), n_ctrl), "strange dominator");
-#ifdef ASSERT
- if (trace) { tty->print("Next mem is"); mem->dump(); }
-#endif
- } while (mem->bottom_type() != Type::MEMORY || !phase->is_dominator(phase->ctrl_or_self(mem), n_ctrl));
-
- assert(mem->bottom_type() == Type::MEMORY, "bad mem");
-
- bool not_dom = false;
- for (uint i = 0; i < phis.size() && !not_dom; i++) {
- Node* nn = phis.at(i);
-
-#ifdef ASSERT
- if (trace) { tty->print("Looking from phi"); nn->dump(); }
-#endif
- assert(nn->is_Phi(), "phis only");
- for (uint j = 2; j < nn->req() && !not_dom; j++) {
- Node* m = nn->in(j);
-#ifdef ASSERT
- if (trace) { tty->print("Input %d is", j); m->dump(); }
-#endif
- while (m != mem && !seen.test_set(m->_idx)) {
- if (is_dominator(phase->ctrl_or_self(m), phase->ctrl_or_self(mem), m, mem, phase)) {
- not_dom = true;
- // Scheduling anomaly
-#ifdef ASSERT
- if (trace) { tty->print("Giving up"); m->dump(); }
-#endif
- break;
- }
- if (!is_independent(m)) {
- if (trace) { tty->print_cr("Not independent"); }
- if (new_mem != in(MemNode::Memory)) {
-#ifdef ASSERT
- if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); }
-#endif
- phase->igvn().replace_input_of(this, MemNode::Memory, new_mem);
- }
- return;
- }
- if (m->is_Phi()) {
- phis.push(m);
- }
- m = next_mem(m, alias);
-#ifdef ASSERT
- if (trace) { tty->print("Next mem is"); m->dump(); }
-#endif
- }
- }
- }
- if (!not_dom) {
- new_mem = mem;
- phis.clear();
- } else {
- seen.Clear();
- }
- }
-}
-
-CallStaticJavaNode* ShenandoahWriteBarrierNode::pin_and_expand_null_check(PhaseIterGVN& igvn) {
- Node* val = in(ValueIn);
-
- const Type* val_t = igvn.type(val);
-
- if (val_t->meet(TypePtr::NULL_PTR) != val_t &&
- val->Opcode() == Op_CastPP &&
- val->in(0) != NULL &&
- val->in(0)->Opcode() == Op_IfTrue &&
- val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) &&
- val->in(0)->in(0)->is_If() &&
- val->in(0)->in(0)->in(1)->Opcode() == Op_Bool &&
- val->in(0)->in(0)->in(1)->as_Bool()->_test._test == BoolTest::ne &&
- val->in(0)->in(0)->in(1)->in(1)->Opcode() == Op_CmpP &&
- val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1) &&
- val->in(0)->in(0)->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) {
- assert(val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1), "");
- CallStaticJavaNode* unc = val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
- return unc;
- }
- return NULL;
-}
-
-void ShenandoahWriteBarrierNode::pin_and_expand_move_barrier(PhaseIdealLoop* phase, GrowableArray<MemoryGraphFixer*>& memory_graph_fixers, Unique_Node_List& uses) {
- Node* unc = pin_and_expand_null_check(phase->igvn());
- Node* val = in(ValueIn);
-
- if (unc != NULL) {
- Node* ctrl = phase->get_ctrl(this);
- Node* unc_ctrl = val->in(0);
-
- // Don't move write barrier in a loop
- IdealLoopTree* loop = phase->get_loop(ctrl);
- IdealLoopTree* unc_loop = phase->get_loop(unc_ctrl);
-
- if (!unc_loop->is_member(loop)) {
- return;
- }
-
- Node* branch = no_branches(ctrl, unc_ctrl, false, phase);
- assert(branch == NULL || branch == NodeSentinel, "was not looking for a branch");
- if (branch == NodeSentinel) {
- return;
- }
-
- RegionNode* r = new RegionNode(3);
- IfNode* iff = unc_ctrl->in(0)->as_If();
-
- Node* ctrl_use = unc_ctrl->unique_ctrl_out();
- Node* unc_ctrl_clone = unc_ctrl->clone();
- phase->register_control(unc_ctrl_clone, loop, iff);
- Node* c = unc_ctrl_clone;
- Node* new_cast = clone_null_check(c, val, unc_ctrl_clone, phase);
- r->init_req(1, new_cast->in(0)->in(0)->as_If()->proj_out(0));
-
- phase->igvn().replace_input_of(unc_ctrl, 0, c->in(0));
- phase->set_idom(unc_ctrl, c->in(0), phase->dom_depth(unc_ctrl));
- phase->lazy_replace(c, unc_ctrl);
- c = NULL;;
- phase->igvn().replace_input_of(val, 0, unc_ctrl_clone);
- phase->set_ctrl(val, unc_ctrl_clone);
-
- IfNode* new_iff = new_cast->in(0)->in(0)->as_If();
- fix_null_check(unc, unc_ctrl_clone, r, uses, phase);
- Node* iff_proj = iff->proj_out(0);
- r->init_req(2, iff_proj);
- phase->register_control(r, phase->ltree_root(), iff);
-
- Node* new_bol = new_iff->in(1)->clone();
- Node* new_cmp = new_bol->in(1)->clone();
- assert(new_cmp->Opcode() == Op_CmpP, "broken");
- assert(new_cmp->in(1) == val->in(1), "broken");
- new_bol->set_req(1, new_cmp);
- new_cmp->set_req(1, this);
- phase->register_new_node(new_bol, new_iff->in(0));
- phase->register_new_node(new_cmp, new_iff->in(0));
- phase->igvn().replace_input_of(new_iff, 1, new_bol);
- phase->igvn().replace_input_of(new_cast, 1, this);
-
- for (DUIterator_Fast imax, i = this->fast_outs(imax); i < imax; i++) {
- Node* u = this->fast_out(i);
- if (u == new_cast || u->Opcode() == Op_ShenandoahWBMemProj || u == new_cmp) {
- continue;
- }
- phase->igvn().rehash_node_delayed(u);
- int nb = u->replace_edge(this, new_cast);
- assert(nb > 0, "no update?");
- --i; imax -= nb;
- }
-
- for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
- Node* u = val->fast_out(i);
- if (u == this) {
- continue;
- }
- phase->igvn().rehash_node_delayed(u);
- int nb = u->replace_edge(val, new_cast);
- assert(nb > 0, "no update?");
- --i; imax -= nb;
- }
-
- Node* new_ctrl = unc_ctrl_clone;
-
- int alias = phase->C->get_alias_index(adr_type());
- MemoryGraphFixer* fixer = find_fixer(memory_graph_fixers, alias);
- if (fixer == NULL) {
- fixer = create_fixer(memory_graph_fixers, alias, phase, true);
- }
-
- Node* proj = find_out_with(Op_ShenandoahWBMemProj);
- fixer->remove(proj);
- Node* mem = fixer->find_mem(new_ctrl, NULL);
-
- if (in(Memory) != mem) {
- disconnect_barrier_mem(this, phase->igvn());
- phase->igvn().replace_input_of(this, Memory, mem);
- }
-
- phase->set_ctrl_and_loop(this, new_ctrl);
- phase->igvn().replace_input_of(this, Control, new_ctrl);
- phase->set_ctrl_and_loop(proj, new_ctrl);
-
- fixer->fix_mem(new_ctrl, new_ctrl, mem, mem, proj, uses);
- }
-}
-
-void ShenandoahWriteBarrierNode::pin_and_expand_helper(PhaseIdealLoop* phase) {
- Node* val = in(ValueIn);
- CallStaticJavaNode* unc = pin_and_expand_null_check(phase->igvn());
- Node* rep = this;
- Node* ctrl = phase->get_ctrl(this);
- if (unc != NULL && val->in(0) == ctrl) {
- Node* unc_ctrl = val->in(0);
- IfNode* other_iff = unc_ctrl->unique_ctrl_out()->as_If();
- ProjNode* other_unc_ctrl = other_iff->proj_out(1);
- Node* cast = NULL;
- for (DUIterator_Fast imax, i = other_unc_ctrl->fast_outs(imax); i < imax && cast == NULL; i++) {
- Node* u = other_unc_ctrl->fast_out(i);
- if (u->Opcode() == Op_CastPP && u->in(1) == this) {
- cast = u;
- }
- }
- assert(other_unc_ctrl->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) == unc, "broken");
- rep = cast;
- }
-
- // Replace all uses of barrier's input that are dominated by ctrl
- // with the value returned by the barrier: no need to keep both
- // live.
- for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
- Node* u = val->fast_out(i);
- if (u != this) {
- if (u->is_Phi()) {
- int nb = 0;
- for (uint j = 1; j < u->req(); j++) {
- if (u->in(j) == val) {
- Node* c = u->in(0)->in(j);
- if (phase->is_dominator(ctrl, c)) {
- phase->igvn().replace_input_of(u, j, rep);
- nb++;
- }
- }
- }
- if (nb > 0) {
- imax -= nb;
- --i;
- }
- } else {
- Node* c = phase->ctrl_or_self(u);
- if (is_dominator(ctrl, c, this, u, phase)) {
- phase->igvn().rehash_node_delayed(u);
- int nb = u->replace_edge(val, rep);
- assert(nb > 0, "no update?");
- --i, imax -= nb;
- }
- }
- }
- }
-}
-
-Node* ShenandoahWriteBarrierNode::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
+Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
Node* mem = NULL;
Node* c = ctrl;
do {
@@ -2355,7 +914,7 @@
return mem;
}
-void ShenandoahWriteBarrierNode::follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase) {
+void ShenandoahBarrierC2Support::follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase) {
for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
Node* u = n->fast_out(i);
if (!u->is_CFG() && phase->get_ctrl(u) == ctrl && (!u->is_Phi() || !u->in(0)->is_Loop() || u->in(LoopNode::LoopBackControl) != n)) {
@@ -2375,7 +934,7 @@
inner->clear_strip_mined();
}
-void ShenandoahWriteBarrierNode::test_heap_stable(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl,
+void ShenandoahBarrierC2Support::test_heap_stable(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl,
PhaseIdealLoop* phase) {
IdealLoopTree* loop = phase->get_loop(ctrl);
Node* thread = new ThreadLocalNode();
@@ -2407,7 +966,7 @@
assert(is_heap_stable_test(heap_stable_iff), "Should match the shape");
}
-void ShenandoahWriteBarrierNode::test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase) {
+void ShenandoahBarrierC2Support::test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase) {
const Type* val_t = phase->igvn().type(val);
if (val_t->meet(TypePtr::NULL_PTR) == val_t) {
IdealLoopTree* loop = phase->get_loop(ctrl);
@@ -2424,7 +983,7 @@
}
}
-Node* ShenandoahWriteBarrierNode::clone_null_check(Node*& c, Node* val, Node* unc_ctrl, PhaseIdealLoop* phase) {
+Node* ShenandoahBarrierC2Support::clone_null_check(Node*& c, Node* val, Node* unc_ctrl, PhaseIdealLoop* phase) {
IdealLoopTree *loop = phase->get_loop(c);
Node* iff = unc_ctrl->in(0);
assert(iff->is_If(), "broken");
@@ -2445,7 +1004,7 @@
return val;
}
-void ShenandoahWriteBarrierNode::fix_null_check(Node* unc, Node* unc_ctrl, Node* new_unc_ctrl,
+void ShenandoahBarrierC2Support::fix_null_check(Node* unc, Node* unc_ctrl, Node* new_unc_ctrl,
Unique_Node_List& uses, PhaseIdealLoop* phase) {
IfNode* iff = unc_ctrl->in(0)->as_If();
Node* proj = iff->proj_out(0);
@@ -2494,7 +1053,7 @@
assert(nb == 1, "only use expected");
}
-void ShenandoahWriteBarrierNode::in_cset_fast_test(Node*& ctrl, Node*& not_cset_ctrl, Node* val, Node* raw_mem, PhaseIdealLoop* phase) {
+void ShenandoahBarrierC2Support::in_cset_fast_test(Node*& ctrl, Node*& not_cset_ctrl, Node* val, Node* raw_mem, PhaseIdealLoop* phase) {
IdealLoopTree *loop = phase->get_loop(ctrl);
Node* raw_rbtrue = new CastP2XNode(ctrl, val);
phase->register_new_node(raw_rbtrue, ctrl);
@@ -2523,23 +1082,18 @@
phase->register_control(ctrl, loop, in_cset_fast_test_iff);
}
-void ShenandoahWriteBarrierNode::call_wb_stub(Node*& ctrl, Node*& val, Node*& result_mem,
- Node* raw_mem, Node* wb_mem,
- int alias,
- PhaseIdealLoop* phase) {
+void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node*& result_mem, Node* raw_mem, PhaseIdealLoop* phase) {
IdealLoopTree*loop = phase->get_loop(ctrl);
const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst();
// The slow path stub consumes and produces raw memory in addition
// to the existing memory edges
Node* base = find_bottom_mem(ctrl, phase);
-
MergeMemNode* mm = MergeMemNode::make(base);
- mm->set_memory_at(alias, wb_mem);
mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
phase->register_new_node(mm, ctrl);
- Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_write_barrier_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT), "shenandoah_write_barrier", TypeRawPtr::BOTTOM);
+ Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_write_barrier_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT), "shenandoah_write_barrier", TypeRawPtr::BOTTOM);
call->init_req(TypeFunc::Control, ctrl);
call->init_req(TypeFunc::I_O, phase->C->top());
call->init_req(TypeFunc::Memory, mm);
@@ -2557,7 +1111,7 @@
phase->register_new_node(val, ctrl);
}
-void ShenandoahWriteBarrierNode::fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase) {
+void ShenandoahBarrierC2Support::fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase) {
Node* ctrl = phase->get_ctrl(barrier);
Node* init_raw_mem = fixer.find_mem(ctrl, barrier);
@@ -2610,26 +1164,32 @@
}
}
-void ShenandoahWriteBarrierNode::pin_and_expand(PhaseIdealLoop* phase) {
- Node_List enqueue_barriers;
- if (ShenandoahStoreValEnqueueBarrier) {
- Unique_Node_List wq;
- wq.push(phase->C->root());
- for (uint i = 0; i < wq.size(); i++) {
- Node* n = wq.at(i);
- if (n->Opcode() == Op_ShenandoahEnqueueBarrier) {
- enqueue_barriers.push(n);
- }
- for (uint i = 0; i < n->req(); i++) {
- Node* in = n->in(i);
- if (in != NULL) {
- wq.push(in);
- }
- }
+static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections& projs, PhaseIdealLoop* phase) {
+ Node* region = NULL;
+ while (c != ctrl) {
+ if (c->is_Region()) {
+ region = c;
+ }
+ c = phase->idom(c);
+ }
+ assert(region != NULL, "");
+ Node* phi = new PhiNode(region, n->bottom_type());
+ for (uint j = 1; j < region->req(); j++) {
+ Node* in = region->in(j);
+ if (phase->is_dominator(projs.fallthrough_catchproj, in)) {
+ phi->init_req(j, n);
+ } else if (phase->is_dominator(projs.catchall_catchproj, in)) {
+ phi->init_req(j, n_clone);
+ } else {
+ phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
}
}
+ phase->register_new_node(phi, region);
+ return phi;
+}
- const bool trace = false;
+void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
+ ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
// Collect raw memory state at CFG points in the entire graph and
// record it in memory_nodes. Optimize the raw memory graph in the
@@ -2637,34 +1197,9 @@
// simpler.
GrowableArray<MemoryGraphFixer*> memory_graph_fixers;
- // Let's try to common write barriers again
- optimize_before_expansion(phase, memory_graph_fixers, true);
-
Unique_Node_List uses;
- for (int i = 0; i < ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i++) {
- ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i);
- Node* ctrl = phase->get_ctrl(wb);
-
- Node* val = wb->in(ValueIn);
- if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) {
- assert(is_dominator(phase->get_ctrl(val), ctrl->in(0)->in(0), val, ctrl->in(0), phase), "can't move");
- phase->set_ctrl(wb, ctrl->in(0)->in(0));
- } else if (ctrl->is_CallRuntime()) {
- assert(is_dominator(phase->get_ctrl(val), ctrl->in(0), val, ctrl, phase), "can't move");
- phase->set_ctrl(wb, ctrl->in(0));
- }
-
- assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "only for write barriers");
- // Look for a null check that dominates this barrier and move the
- // barrier right after the null check to enable implicit null
- // checks
- wb->pin_and_expand_move_barrier(phase, memory_graph_fixers, uses);
-
- wb->pin_and_expand_helper(phase);
- }
-
- for (uint i = 0; i < enqueue_barriers.size(); i++) {
- Node* barrier = enqueue_barriers.at(i);
+ for (int i = 0; i < state->enqueue_barriers_count(); i++) {
+ Node* barrier = state->enqueue_barrier(i);
Node* ctrl = phase->get_ctrl(barrier);
IdealLoopTree* loop = phase->get_loop(ctrl);
if (loop->_head->is_OuterStripMinedLoop()) {
@@ -2676,24 +1211,386 @@
}
}
- for (int i = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i > 0; i--) {
- int cnt = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count();
- ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i-1);
- Node* ctrl = phase->get_ctrl(wb);
- IdealLoopTree* loop = phase->get_loop(ctrl);
- if (loop->_head->is_OuterStripMinedLoop()) {
- // Expanding a barrier here will break loop strip mining
- // verification. Transform the loop so the loop nest doesn't
- // appear as strip mined.
- OuterStripMinedLoopNode* outer = loop->_head->as_OuterStripMinedLoop();
- hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase);
+ Node_Stack stack(0);
+ Node_List clones;
+ for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
+ ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
+ if (lrb->get_barrier_strength() == ShenandoahLoadReferenceBarrierNode::NONE) {
+ continue;
+ }
+
+ Node* ctrl = phase->get_ctrl(lrb);
+ Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
+
+ CallStaticJavaNode* unc = NULL;
+ Node* unc_ctrl = NULL;
+ Node* uncasted_val = val;
+
+ for (DUIterator_Fast imax, i = lrb->fast_outs(imax); i < imax; i++) {
+ Node* u = lrb->fast_out(i);
+ if (u->Opcode() == Op_CastPP &&
+ u->in(0) != NULL &&
+ phase->is_dominator(u->in(0), ctrl)) {
+ const Type* u_t = phase->igvn().type(u);
+
+ if (u_t->meet(TypePtr::NULL_PTR) != u_t &&
+ u->in(0)->Opcode() == Op_IfTrue &&
+ u->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) &&
+ u->in(0)->in(0)->is_If() &&
+ u->in(0)->in(0)->in(1)->Opcode() == Op_Bool &&
+ u->in(0)->in(0)->in(1)->as_Bool()->_test._test == BoolTest::ne &&
+ u->in(0)->in(0)->in(1)->in(1)->Opcode() == Op_CmpP &&
+ u->in(0)->in(0)->in(1)->in(1)->in(1) == val &&
+ u->in(0)->in(0)->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) {
+ IdealLoopTree* loop = phase->get_loop(ctrl);
+ IdealLoopTree* unc_loop = phase->get_loop(u->in(0));
+
+ if (!unc_loop->is_member(loop)) {
+ continue;
+ }
+
+ Node* branch = no_branches(ctrl, u->in(0), false, phase);
+ assert(branch == NULL || branch == NodeSentinel, "was not looking for a branch");
+ if (branch == NodeSentinel) {
+ continue;
+ }
+
+ phase->igvn().replace_input_of(u, 1, val);
+ phase->igvn().replace_input_of(lrb, ShenandoahLoadReferenceBarrierNode::ValueIn, u);
+ phase->set_ctrl(u, u->in(0));
+ phase->set_ctrl(lrb, u->in(0));
+ unc = u->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
+ unc_ctrl = u->in(0);
+ val = u;
+
+ for (DUIterator_Fast jmax, j = val->fast_outs(jmax); j < jmax; j++) {
+ Node* u = val->fast_out(j);
+ if (u == lrb) continue;
+ phase->igvn().rehash_node_delayed(u);
+ int nb = u->replace_edge(val, lrb);
+ --j; jmax -= nb;
+ }
+
+ RegionNode* r = new RegionNode(3);
+ IfNode* iff = unc_ctrl->in(0)->as_If();
+
+ Node* ctrl_use = unc_ctrl->unique_ctrl_out();
+ Node* unc_ctrl_clone = unc_ctrl->clone();
+ phase->register_control(unc_ctrl_clone, loop, iff);
+ Node* c = unc_ctrl_clone;
+ Node* new_cast = clone_null_check(c, val, unc_ctrl_clone, phase);
+ r->init_req(1, new_cast->in(0)->in(0)->as_If()->proj_out(0));
+
+ phase->igvn().replace_input_of(unc_ctrl, 0, c->in(0));
+ phase->set_idom(unc_ctrl, c->in(0), phase->dom_depth(unc_ctrl));
+ phase->lazy_replace(c, unc_ctrl);
+ c = NULL;;
+ phase->igvn().replace_input_of(val, 0, unc_ctrl_clone);
+ phase->set_ctrl(val, unc_ctrl_clone);
+
+ IfNode* new_iff = new_cast->in(0)->in(0)->as_If();
+ fix_null_check(unc, unc_ctrl_clone, r, uses, phase);
+ Node* iff_proj = iff->proj_out(0);
+ r->init_req(2, iff_proj);
+ phase->register_control(r, phase->ltree_root(), iff);
+
+ Node* new_bol = new_iff->in(1)->clone();
+ Node* new_cmp = new_bol->in(1)->clone();
+ assert(new_cmp->Opcode() == Op_CmpP, "broken");
+ assert(new_cmp->in(1) == val->in(1), "broken");
+ new_bol->set_req(1, new_cmp);
+ new_cmp->set_req(1, lrb);
+ phase->register_new_node(new_bol, new_iff->in(0));
+ phase->register_new_node(new_cmp, new_iff->in(0));
+ phase->igvn().replace_input_of(new_iff, 1, new_bol);
+ phase->igvn().replace_input_of(new_cast, 1, lrb);
+
+ for (DUIterator_Fast imax, i = lrb->fast_outs(imax); i < imax; i++) {
+ Node* u = lrb->fast_out(i);
+ if (u == new_cast || u == new_cmp) {
+ continue;
+ }
+ phase->igvn().rehash_node_delayed(u);
+ int nb = u->replace_edge(lrb, new_cast);
+ assert(nb > 0, "no update?");
+ --i; imax -= nb;
+ }
+
+ for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
+ Node* u = val->fast_out(i);
+ if (u == lrb) {
+ continue;
+ }
+ phase->igvn().rehash_node_delayed(u);
+ int nb = u->replace_edge(val, new_cast);
+ assert(nb > 0, "no update?");
+ --i; imax -= nb;
+ }
+
+ ctrl = unc_ctrl_clone;
+ phase->set_ctrl_and_loop(lrb, ctrl);
+ break;
+ }
+ }
+ }
+ if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) {
+ CallNode* call = ctrl->in(0)->as_CallJava();
+ CallProjections projs;
+ call->extract_projections(&projs, false, false);
+
+ Node* lrb_clone = lrb->clone();
+ phase->register_new_node(lrb_clone, projs.catchall_catchproj);
+ phase->set_ctrl(lrb, projs.fallthrough_catchproj);
+
+ stack.push(lrb, 0);
+ clones.push(lrb_clone);
+
+ do {
+ assert(stack.size() == clones.size(), "");
+ Node* n = stack.node();
+#ifdef ASSERT
+ if (n->is_Load()) {
+ Node* mem = n->in(MemNode::Memory);
+ for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
+ Node* u = mem->fast_out(j);
+ assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
+ }
+ }
+#endif
+ uint idx = stack.index();
+ Node* n_clone = clones.at(clones.size()-1);
+ if (idx < n->outcnt()) {
+ Node* u = n->raw_out(idx);
+ Node* c = phase->ctrl_or_self(u);
+ if (c == ctrl) {
+ stack.set_index(idx+1);
+ assert(!u->is_CFG(), "");
+ stack.push(u, 0);
+ Node* u_clone = u->clone();
+ int nb = u_clone->replace_edge(n, n_clone);
+ assert(nb > 0, "should have replaced some uses");
+ phase->register_new_node(u_clone, projs.catchall_catchproj);
+ clones.push(u_clone);
+ phase->set_ctrl(u, projs.fallthrough_catchproj);
+ } else {
+ bool replaced = false;
+ if (u->is_Phi()) {
+ for (uint k = 1; k < u->req(); k++) {
+ if (u->in(k) == n) {
+ if (phase->is_dominator(projs.catchall_catchproj, u->in(0)->in(k))) {
+ phase->igvn().replace_input_of(u, k, n_clone);
+ replaced = true;
+ } else if (!phase->is_dominator(projs.fallthrough_catchproj, u->in(0)->in(k))) {
+ phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
+ replaced = true;
+ }
+ }
+ }
+ } else {
+ if (phase->is_dominator(projs.catchall_catchproj, c)) {
+ phase->igvn().rehash_node_delayed(u);
+ int nb = u->replace_edge(n, n_clone);
+ assert(nb > 0, "should have replaced some uses");
+ replaced = true;
+ } else if (!phase->is_dominator(projs.fallthrough_catchproj, c)) {
+ phase->igvn().rehash_node_delayed(u);
+ int nb = u->replace_edge(n, create_phis_on_call_return(ctrl, c, n, n_clone, projs, phase));
+ assert(nb > 0, "should have replaced some uses");
+ replaced = true;
+ }
+ }
+ if (!replaced) {
+ stack.set_index(idx+1);
+ }
+ }
+ } else {
+ // assert(n_clone->outcnt() > 0, "");
+ // assert(n->outcnt() > 0, "");
+ stack.pop();
+ clones.pop();
+ }
+ } while (stack.size() > 0);
+ assert(stack.size() == 0 && clones.size() == 0, "");
+ ctrl = projs.fallthrough_catchproj;
}
}
+ // Expand load-reference-barriers
MemoryGraphFixer fixer(Compile::AliasIdxRaw, true, phase);
Unique_Node_List uses_to_ignore;
- for (uint i = 0; i < enqueue_barriers.size(); i++) {
- Node* barrier = enqueue_barriers.at(i);
+ for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
+ ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
+ if (lrb->get_barrier_strength() == ShenandoahLoadReferenceBarrierNode::NONE) {
+ phase->igvn().replace_node(lrb, lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn));
+ continue;
+ }
+ uint last = phase->C->unique();
+ Node* ctrl = phase->get_ctrl(lrb);
+ Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
+
+
+ Node* orig_ctrl = ctrl;
+
+ Node* raw_mem = fixer.find_mem(ctrl, lrb);
+ Node* init_raw_mem = raw_mem;
+ Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL);
+ // int alias = phase->C->get_alias_index(lrb->adr_type());
+
+ IdealLoopTree *loop = phase->get_loop(ctrl);
+ CallStaticJavaNode* unc = lrb->pin_and_expand_null_check(phase->igvn());
+ Node* unc_ctrl = NULL;
+ if (unc != NULL) {
+ if (val->in(ShenandoahLoadReferenceBarrierNode::Control) != ctrl) {
+ unc = NULL;
+ } else {
+ unc_ctrl = val->in(ShenandoahLoadReferenceBarrierNode::Control);
+ }
+ }
+
+ Node* uncasted_val = val;
+ if (unc != NULL) {
+ uncasted_val = val->in(1);
+ }
+
+ Node* heap_stable_ctrl = NULL;
+ Node* null_ctrl = NULL;
+
+ assert(val->bottom_type()->make_oopptr(), "need oop");
+ assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant");
+
+ enum { _heap_stable = 1, _not_cset, _not_equal, _evac_path, _null_path, PATH_LIMIT };
+ Node* region = new RegionNode(PATH_LIMIT);
+ Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr());
+ Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
+
+ // Stable path.
+ test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase);
+ IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If();
+
+ // Heap stable case
+ region->init_req(_heap_stable, heap_stable_ctrl);
+ val_phi->init_req(_heap_stable, uncasted_val);
+ raw_mem_phi->init_req(_heap_stable, raw_mem);
+
+ Node* reg2_ctrl = NULL;
+ // Null case
+ test_null(ctrl, val, null_ctrl, phase);
+ if (null_ctrl != NULL) {
+ reg2_ctrl = null_ctrl->in(0);
+ region->init_req(_null_path, null_ctrl);
+ val_phi->init_req(_null_path, uncasted_val);
+ raw_mem_phi->init_req(_null_path, raw_mem);
+ } else {
+ region->del_req(_null_path);
+ val_phi->del_req(_null_path);
+ raw_mem_phi->del_req(_null_path);
+ }
+
+ // Test for in-cset.
+ // Wires !in_cset(obj) to slot 2 of region and phis
+ Node* not_cset_ctrl = NULL;
+ in_cset_fast_test(ctrl, not_cset_ctrl, uncasted_val, raw_mem, phase);
+ if (not_cset_ctrl != NULL) {
+ if (reg2_ctrl == NULL) reg2_ctrl = not_cset_ctrl->in(0);
+ region->init_req(_not_cset, not_cset_ctrl);
+ val_phi->init_req(_not_cset, uncasted_val);
+ raw_mem_phi->init_req(_not_cset, raw_mem);
+ }
+
+ // Resolve object when orig-value is in cset.
+ // Make the unconditional resolve for fwdptr.
+ Node* new_val = uncasted_val;
+ if (unc_ctrl != NULL) {
+ // Clone the null check in this branch to allow implicit null check
+ new_val = clone_null_check(ctrl, val, unc_ctrl, phase);
+ fix_null_check(unc, unc_ctrl, ctrl->in(0)->as_If()->proj_out(0), uses, phase);
+
+ IfNode* iff = unc_ctrl->in(0)->as_If();
+ phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1));
+ }
+ Node* addr = new AddPNode(new_val, uncasted_val, phase->igvn().MakeConX(ShenandoahBrooksPointer::byte_offset()));
+ phase->register_new_node(addr, ctrl);
+ assert(val->bottom_type()->isa_oopptr(), "what else?");
+ const TypePtr* obj_type = val->bottom_type()->is_oopptr();
+ const TypePtr* adr_type = TypeRawPtr::BOTTOM;
+ Node* fwd = new LoadPNode(ctrl, raw_mem, addr, adr_type, obj_type, MemNode::unordered);
+ phase->register_new_node(fwd, ctrl);
+
+ // Only branch to LRB stub if object is not forwarded; otherwise reply with fwd ptr
+ Node* cmp = new CmpPNode(fwd, new_val);
+ phase->register_new_node(cmp, ctrl);
+ Node* bol = new BoolNode(cmp, BoolTest::eq);
+ phase->register_new_node(bol, ctrl);
+
+ IfNode* iff = new IfNode(ctrl, bol, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
+ if (reg2_ctrl == NULL) reg2_ctrl = iff;
+ phase->register_control(iff, loop, ctrl);
+ Node* if_not_eq = new IfFalseNode(iff);
+ phase->register_control(if_not_eq, loop, iff);
+ Node* if_eq = new IfTrueNode(iff);
+ phase->register_control(if_eq, loop, iff);
+
+ // Wire up not-equal-path in slots 3.
+ region->init_req(_not_equal, if_not_eq);
+ val_phi->init_req(_not_equal, fwd);
+ raw_mem_phi->init_req(_not_equal, raw_mem);
+
+ // Call wb-stub and wire up that path in slots 4
+ Node* result_mem = NULL;
+ ctrl = if_eq;
+ call_lrb_stub(ctrl, fwd, result_mem, raw_mem, phase);
+ region->init_req(_evac_path, ctrl);
+ val_phi->init_req(_evac_path, fwd);
+ raw_mem_phi->init_req(_evac_path, result_mem);
+
+ phase->register_control(region, loop, heap_stable_iff);
+ Node* out_val = val_phi;
+ phase->register_new_node(val_phi, region);
+ phase->register_new_node(raw_mem_phi, region);
+
+ fix_ctrl(lrb, region, fixer, uses, uses_to_ignore, last, phase);
+
+ ctrl = orig_ctrl;
+
+ if (unc != NULL) {
+ for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
+ Node* u = val->fast_out(i);
+ Node* c = phase->ctrl_or_self(u);
+ if (u != lrb && (c != ctrl || is_dominator_same_ctrl(c, lrb, u, phase))) {
+ phase->igvn().rehash_node_delayed(u);
+ int nb = u->replace_edge(val, out_val);
+ --i, imax -= nb;
+ }
+ }
+ if (val->outcnt() == 0) {
+ phase->igvn()._worklist.push(val);
+ }
+ }
+ phase->igvn().replace_node(lrb, out_val);
+
+ follow_barrier_uses(out_val, ctrl, uses, phase);
+
+ for(uint next = 0; next < uses.size(); next++ ) {
+ Node *n = uses.at(next);
+ assert(phase->get_ctrl(n) == ctrl, "bad control");
+ assert(n != init_raw_mem, "should leave input raw mem above the barrier");
+ phase->set_ctrl(n, region);
+ follow_barrier_uses(n, ctrl, uses, phase);
+ }
+
+ // The slow path call produces memory: hook the raw memory phi
+ // from the expanded load reference barrier with the rest of the graph
+ // which may require adding memory phis at every post dominated
+ // region and at enclosing loop heads. Use the memory state
+ // collected in memory_nodes to fix the memory graph. Update that
+ // memory state as we go.
+ fixer.fix_mem(ctrl, region, init_raw_mem, raw_mem_for_ctrl, raw_mem_phi, uses);
+ }
+ // Done expanding load-reference-barriers.
+ assert(ShenandoahBarrierSetC2::bsc2()->state()->load_reference_barriers_count() == 0, "all load reference barrier nodes should have been replaced");
+
+ for (int i = state->enqueue_barriers_count() - 1; i >= 0; i--) {
+ Node* barrier = state->enqueue_barrier(i);
Node* pre_val = barrier->in(1);
if (phase->igvn().type(pre_val)->higher_equal(TypePtr::NULL_PTR)) {
@@ -2840,212 +1737,11 @@
phase->igvn().replace_node(barrier, pre_val);
}
-
- for (int i = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i > 0; i--) {
- int cnt = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count();
- ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i-1);
-
- uint last = phase->C->unique();
- Node* ctrl = phase->get_ctrl(wb);
- Node* orig_ctrl = ctrl;
-
- Node* raw_mem = fixer.find_mem(ctrl, wb);
- Node* init_raw_mem = raw_mem;
- Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL);
- int alias = phase->C->get_alias_index(wb->adr_type());
- Node* wb_mem = wb->in(Memory);
- Node* init_wb_mem = wb_mem;
-
- Node* val = wb->in(ValueIn);
- Node* wbproj = wb->find_out_with(Op_ShenandoahWBMemProj);
- IdealLoopTree *loop = phase->get_loop(ctrl);
-
- assert(val->Opcode() != Op_ShenandoahWriteBarrier, "No chain of write barriers");
-
- CallStaticJavaNode* unc = wb->pin_and_expand_null_check(phase->igvn());
- Node* unc_ctrl = NULL;
- if (unc != NULL) {
- if (val->in(0) != ctrl) {
- unc = NULL;
- } else {
- unc_ctrl = val->in(0);
- }
- }
-
- Node* uncasted_val = val;
- if (unc != NULL) {
- uncasted_val = val->in(1);
- }
-
- Node* heap_stable_ctrl = NULL;
- Node* null_ctrl = NULL;
-
- assert(val->bottom_type()->make_oopptr(), "need oop");
- assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant");
-
- enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
- Node* region = new RegionNode(PATH_LIMIT);
- Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr());
- Node* mem_phi = PhiNode::make(region, wb_mem, Type::MEMORY, phase->C->alias_type(wb->adr_type())->adr_type());
- Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
-
- enum { _not_cset = 1, _not_equal, _evac_path, _null_path, PATH_LIMIT2 };
- Node* region2 = new RegionNode(PATH_LIMIT2);
- Node* val_phi2 = new PhiNode(region2, uncasted_val->bottom_type()->is_oopptr());
- Node* mem_phi2 = PhiNode::make(region2, wb_mem, Type::MEMORY, phase->C->alias_type(wb->adr_type())->adr_type());
- Node* raw_mem_phi2 = PhiNode::make(region2, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
-
- // Stable path.
- test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase);
- IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If();
-
- // Heap stable case
- region->init_req(_heap_stable, heap_stable_ctrl);
- val_phi->init_req(_heap_stable, uncasted_val);
- mem_phi->init_req(_heap_stable, wb_mem);
- raw_mem_phi->init_req(_heap_stable, raw_mem);
-
- Node* reg2_ctrl = NULL;
- // Null case
- test_null(ctrl, val, null_ctrl, phase);
- if (null_ctrl != NULL) {
- reg2_ctrl = null_ctrl->in(0);
- region2->init_req(_null_path, null_ctrl);
- val_phi2->init_req(_null_path, uncasted_val);
- mem_phi2->init_req(_null_path, wb_mem);
- raw_mem_phi2->init_req(_null_path, raw_mem);
- } else {
- region2->del_req(_null_path);
- val_phi2->del_req(_null_path);
- mem_phi2->del_req(_null_path);
- raw_mem_phi2->del_req(_null_path);
- }
-
- // Test for in-cset.
- // Wires !in_cset(obj) to slot 2 of region and phis
- Node* not_cset_ctrl = NULL;
- in_cset_fast_test(ctrl, not_cset_ctrl, uncasted_val, raw_mem, phase);
- if (not_cset_ctrl != NULL) {
- if (reg2_ctrl == NULL) reg2_ctrl = not_cset_ctrl->in(0);
- region2->init_req(_not_cset, not_cset_ctrl);
- val_phi2->init_req(_not_cset, uncasted_val);
- mem_phi2->init_req(_not_cset, wb_mem);
- raw_mem_phi2->init_req(_not_cset, raw_mem);
- }
-
- // Resolve object when orig-value is in cset.
- // Make the unconditional resolve for fwdptr, not the read barrier.
- Node* new_val = uncasted_val;
- if (unc_ctrl != NULL) {
- // Clone the null check in this branch to allow implicit null check
- new_val = clone_null_check(ctrl, val, unc_ctrl, phase);
- fix_null_check(unc, unc_ctrl, ctrl->in(0)->as_If()->proj_out(0), uses, phase);
+ assert(state->enqueue_barriers_count() == 0, "all enqueue barrier nodes should have been replaced");
- IfNode* iff = unc_ctrl->in(0)->as_If();
- phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1));
- }
- Node* addr = new AddPNode(new_val, uncasted_val, phase->igvn().MakeConX(ShenandoahBrooksPointer::byte_offset()));
- phase->register_new_node(addr, ctrl);
- assert(val->bottom_type()->isa_oopptr(), "what else?");
- const TypePtr* obj_type = val->bottom_type()->is_oopptr();
- const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type);
- Node* fwd = new LoadPNode(ctrl, wb_mem, addr, adr_type, obj_type, MemNode::unordered);
- phase->register_new_node(fwd, ctrl);
-
- // Only branch to WB stub if object is not forwarded; otherwise reply with fwd ptr
- Node* cmp = new CmpPNode(fwd, new_val);
- phase->register_new_node(cmp, ctrl);
- Node* bol = new BoolNode(cmp, BoolTest::eq);
- phase->register_new_node(bol, ctrl);
-
- IfNode* iff = new IfNode(ctrl, bol, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
- if (reg2_ctrl == NULL) reg2_ctrl = iff;
- phase->register_control(iff, loop, ctrl);
- Node* if_not_eq = new IfFalseNode(iff);
- phase->register_control(if_not_eq, loop, iff);
- Node* if_eq = new IfTrueNode(iff);
- phase->register_control(if_eq, loop, iff);
-
- // Wire up not-equal-path in slots 3.
- region2->init_req(_not_equal, if_not_eq);
- val_phi2->init_req(_not_equal, fwd);
- mem_phi2->init_req(_not_equal, wb_mem);
- raw_mem_phi2->init_req(_not_equal, raw_mem);
-
- // Call wb-stub and wire up that path in slots 4
- Node* result_mem = NULL;
- ctrl = if_eq;
- call_wb_stub(ctrl, new_val, result_mem,
- raw_mem, wb_mem,
- alias, phase);
- region2->init_req(_evac_path, ctrl);
- val_phi2->init_req(_evac_path, new_val);
- mem_phi2->init_req(_evac_path, result_mem);
- raw_mem_phi2->init_req(_evac_path, result_mem);
-
- phase->register_control(region2, loop, reg2_ctrl);
- phase->register_new_node(val_phi2, region2);
- phase->register_new_node(mem_phi2, region2);
- phase->register_new_node(raw_mem_phi2, region2);
-
- region->init_req(_heap_unstable, region2);
- val_phi->init_req(_heap_unstable, val_phi2);
- mem_phi->init_req(_heap_unstable, mem_phi2);
- raw_mem_phi->init_req(_heap_unstable, raw_mem_phi2);
-
- phase->register_control(region, loop, heap_stable_iff);
- Node* out_val = val_phi;
- phase->register_new_node(val_phi, region);
- phase->register_new_node(mem_phi, region);
- phase->register_new_node(raw_mem_phi, region);
-
- fix_ctrl(wb, region, fixer, uses, uses_to_ignore, last, phase);
-
- ctrl = orig_ctrl;
-
- phase->igvn().replace_input_of(wbproj, ShenandoahWBMemProjNode::WriteBarrier, phase->C->top());
- phase->igvn().replace_node(wbproj, mem_phi);
- if (unc != NULL) {
- for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
- Node* u = val->fast_out(i);
- Node* c = phase->ctrl_or_self(u);
- if (u != wb && (c != ctrl || is_dominator_same_ctrl(c, wb, u, phase))) {
- phase->igvn().rehash_node_delayed(u);
- int nb = u->replace_edge(val, out_val);
- --i, imax -= nb;
- }
- }
- if (val->outcnt() == 0) {
- phase->igvn()._worklist.push(val);
- }
- }
- phase->igvn().replace_node(wb, out_val);
-
- follow_barrier_uses(mem_phi, ctrl, uses, phase);
- follow_barrier_uses(out_val, ctrl, uses, phase);
-
- for(uint next = 0; next < uses.size(); next++ ) {
- Node *n = uses.at(next);
- assert(phase->get_ctrl(n) == ctrl, "bad control");
- assert(n != init_raw_mem, "should leave input raw mem above the barrier");
- phase->set_ctrl(n, region);
- follow_barrier_uses(n, ctrl, uses, phase);
- }
-
- // The slow path call produces memory: hook the raw memory phi
- // from the expanded write barrier with the rest of the graph
- // which may require adding memory phis at every post dominated
- // region and at enclosing loop heads. Use the memory state
- // collected in memory_nodes to fix the memory graph. Update that
- // memory state as we go.
- fixer.fix_mem(ctrl, region, init_raw_mem, raw_mem_for_ctrl, raw_mem_phi, uses);
- assert(ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() == cnt - 1, "not replaced");
- }
-
- assert(ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() == 0, "all write barrier nodes should have been replaced");
}
-void ShenandoahWriteBarrierNode::move_heap_stable_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) {
+void ShenandoahBarrierC2Support::move_heap_stable_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) {
IdealLoopTree *loop = phase->get_loop(iff);
Node* loop_head = loop->_head;
Node* entry_c = loop_head->in(LoopNode::EntryControl);
@@ -3078,7 +1774,7 @@
}
}
-bool ShenandoahWriteBarrierNode::identical_backtoback_ifs(Node *n, PhaseIdealLoop* phase) {
+bool ShenandoahBarrierC2Support::identical_backtoback_ifs(Node* n, PhaseIdealLoop* phase) {
if (!n->is_If() || n->is_CountedLoopEnd()) {
return false;
}
@@ -3113,7 +1809,7 @@
return true;
}
-void ShenandoahWriteBarrierNode::merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase) {
+void ShenandoahBarrierC2Support::merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase) {
assert(is_heap_stable_test(n), "no other tests");
if (identical_backtoback_ifs(n, phase)) {
Node* n_ctrl = n->in(0);
@@ -3149,7 +1845,7 @@
}
}
-IfNode* ShenandoahWriteBarrierNode::find_unswitching_candidate(const IdealLoopTree *loop, PhaseIdealLoop* phase) {
+IfNode* ShenandoahBarrierC2Support::find_unswitching_candidate(const IdealLoopTree* loop, PhaseIdealLoop* phase) {
// Find first invariant test that doesn't exit the loop
LoopNode *head = loop->_head->as_Loop();
IfNode* unswitch_iff = NULL;
@@ -3194,10 +1890,9 @@
}
-void ShenandoahWriteBarrierNode::optimize_after_expansion(VectorSet &visited, Node_Stack &stack, Node_List &old_new, PhaseIdealLoop* phase) {
+void ShenandoahBarrierC2Support::optimize_after_expansion(VectorSet &visited, Node_Stack &stack, Node_List &old_new, PhaseIdealLoop* phase) {
Node_List heap_stable_tests;
Node_List gc_state_loads;
-
stack.push(phase->C->start(), 0);
do {
Node* n = stack.node();
@@ -3274,7 +1969,7 @@
}
#ifdef ASSERT
-void ShenandoahBarrierNode::verify_raw_mem(RootNode* root) {
+void ShenandoahBarrierC2Support::verify_raw_mem(RootNode* root) {
const bool trace = false;
ResourceMark rm;
Unique_Node_List nodes;
@@ -3372,6 +2067,10 @@
}
#endif
+ShenandoahEnqueueBarrierNode::ShenandoahEnqueueBarrierNode(Node* val) : Node(NULL, val) {
+ ShenandoahBarrierSetC2::bsc2()->state()->add_enqueue_barrier(this);
+}
+
const Type* ShenandoahEnqueueBarrierNode::bottom_type() const {
if (in(1) == NULL || in(1)->is_top()) {
return Type::TOP;
@@ -3531,6 +2230,26 @@
Node* call = in->in(0)->in(0);
assert(call->is_Call(), "");
mem = call->in(TypeFunc::Memory);
+ } else if (in->Opcode() == Op_NeverBranch) {
+ ResourceMark rm;
+ Unique_Node_List wq;
+ wq.push(in);
+ wq.push(in->as_Multi()->proj_out(0));
+ for (uint j = 1; j < wq.size(); j++) {
+ Node* c = wq.at(j);
+ assert(!c->is_Root(), "shouldn't leave loop");
+ if (c->is_SafePoint()) {
+ assert(mem == NULL, "only one safepoint");
+ mem = c->in(TypeFunc::Memory);
+ }
+ for (DUIterator_Fast kmax, k = c->fast_outs(kmax); k < kmax; k++) {
+ Node* u = c->fast_out(k);
+ if (u->is_CFG()) {
+ wq.push(u);
+ }
+ }
+ }
+ assert(mem != NULL, "should have found safepoint");
}
}
} else {
@@ -3569,12 +2288,6 @@
assert(_alias == Compile::AliasIdxRaw, "");
stack.push(mem, mem->req());
mem = mem->in(MemNode::Memory);
- } else if (mem->Opcode() == Op_ShenandoahWriteBarrier) {
- assert(_alias != Compile::AliasIdxRaw, "");
- mem = mem->in(ShenandoahBarrierNode::Memory);
- } else if (mem->Opcode() == Op_ShenandoahWBMemProj) {
- stack.push(mem, mem->req());
- mem = mem->in(ShenandoahWBMemProjNode::WriteBarrier);
} else {
#ifdef ASSERT
mem->dump();
@@ -3628,7 +2341,7 @@
while (progress) {
progress = false;
iteration++;
- assert(iteration <= 2+max_depth || _phase->C->has_irreducible_loop(), "");
+ assert(iteration <= 2+max_depth || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "");
if (trace) { tty->print_cr("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); }
IdealLoopTree* last_updated_ilt = NULL;
for (int i = rpo_list.size() - 1; i >= 0; i--) {
@@ -3796,7 +2509,7 @@
mem = _memory_nodes[c->_idx];
}
if (n != NULL && mem_is_valid(mem, c)) {
- while (!ShenandoahWriteBarrierNode::is_dominator_same_ctrl(c, mem, n, _phase) && _phase->ctrl_or_self(mem) == ctrl) {
+ while (!ShenandoahBarrierC2Support::is_dominator_same_ctrl(c, mem, n, _phase) && _phase->ctrl_or_self(mem) == ctrl) {
mem = next_mem(mem, _alias);
}
if (mem->is_MergeMem()) {
@@ -3842,12 +2555,6 @@
} else if (old->Opcode() == Op_SCMemProj) {
assert(_alias == Compile::AliasIdxRaw, "");
old = old->in(0);
- } else if (old->Opcode() == Op_ShenandoahWBMemProj) {
- assert(_alias != Compile::AliasIdxRaw, "");
- old = old->in(ShenandoahWBMemProjNode::WriteBarrier);
- } else if (old->Opcode() == Op_ShenandoahWriteBarrier) {
- assert(_alias != Compile::AliasIdxRaw, "");
- old = old->in(ShenandoahBarrierNode::Memory);
} else {
ShouldNotReachHere();
}
@@ -3857,7 +2564,7 @@
_memory_nodes.map(ctrl->_idx, mem);
_memory_nodes.map(new_ctrl->_idx, mem_for_ctrl);
}
- uint input = prev->Opcode() == Op_ShenandoahWriteBarrier ? (uint)ShenandoahBarrierNode::Memory : (uint)MemNode::Memory;
+ uint input = (uint)MemNode::Memory;
_phase->igvn().replace_input_of(prev, input, new_mem);
} else {
uses.clear();
@@ -3925,19 +2632,14 @@
} else {
DEBUG_ONLY(if (trace) { tty->print("ZZZ NOT setting mem"); m->dump(); });
for (;;) {
- assert(m->is_Mem() || m->is_LoadStore() || m->is_Proj() || m->Opcode() == Op_ShenandoahWriteBarrier || m->Opcode() == Op_ShenandoahWBMemProj, "");
+ assert(m->is_Mem() || m->is_LoadStore() || m->is_Proj(), "");
Node* next = NULL;
if (m->is_Proj()) {
next = m->in(0);
- } else if (m->Opcode() == Op_ShenandoahWBMemProj) {
- next = m->in(ShenandoahWBMemProjNode::WriteBarrier);
- } else if (m->is_Mem() || m->is_LoadStore()) {
+ } else {
+ assert(m->is_Mem() || m->is_LoadStore(), "");
assert(_alias == Compile::AliasIdxRaw, "");
next = m->in(MemNode::Memory);
- } else {
- assert(_alias != Compile::AliasIdxRaw, "");
- assert (m->Opcode() == Op_ShenandoahWriteBarrier, "");
- next = m->in(ShenandoahBarrierNode::Memory);
}
if (_phase->get_ctrl(next) != u) {
break;
@@ -3954,8 +2656,8 @@
}
DEBUG_ONLY(if (trace) { tty->print("ZZZ setting to phi"); m->dump(); });
- assert(m->is_Mem() || m->is_LoadStore() || m->Opcode() == Op_ShenandoahWriteBarrier, "");
- uint input = (m->is_Mem() || m->is_LoadStore()) ? (uint)MemNode::Memory : (uint)ShenandoahBarrierNode::Memory;
+ assert(m->is_Mem() || m->is_LoadStore(), "");
+ uint input = (uint)MemNode::Memory;
_phase->igvn().replace_input_of(m, input, phi);
push = false;
}
@@ -4181,20 +2883,7 @@
for (DUIterator i = mem->outs(); mem->has_out(i); i++) {
Node* u = mem->out(i);
if (u != replacement && u->_idx < last) {
- if (u->is_ShenandoahBarrier() && _alias != Compile::AliasIdxRaw) {
- if (_phase->C->get_alias_index(u->adr_type()) == _alias && ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) {
- _phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj);
- assert(u->find_edge(mem) == -1, "only one edge");
- --i;
- }
- } else if (u->is_Mem()) {
- if (_phase->C->get_alias_index(u->adr_type()) == _alias && ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) {
- assert(_alias == Compile::AliasIdxRaw , "only raw memory can lead to a memory operation");
- _phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj);
- assert(u->find_edge(mem) == -1, "only one edge");
- --i;
- }
- } else if (u->is_MergeMem()) {
+ if (u->is_MergeMem()) {
MergeMemNode* u_mm = u->as_MergeMem();
if (u_mm->memory_at(_alias) == mem) {
MergeMemNode* newmm = NULL;
@@ -4222,7 +2911,7 @@
}
}
} else {
- if (rep_ctrl != uu && ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(uu), replacement, uu, _phase)) {
+ if (rep_ctrl != uu && ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(uu), replacement, uu, _phase)) {
if (newmm == NULL) {
newmm = clone_merge_mem(u, mem, rep_proj, rep_ctrl, i);
}
@@ -4263,10 +2952,11 @@
u->Opcode() == Op_Rethrow ||
u->Opcode() == Op_Return ||
u->Opcode() == Op_SafePoint ||
+ u->Opcode() == Op_StoreLConditional ||
(u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) ||
(u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) ||
u->Opcode() == Op_CallLeaf, "");
- if (ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) {
+ if (ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) {
if (mm == NULL) {
mm = allocate_merge_mem(mem, rep_proj, rep_ctrl);
}
@@ -4274,7 +2964,7 @@
--i;
}
} else if (_phase->C->get_alias_index(u->adr_type()) == _alias) {
- if (ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) {
+ if (ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) {
_phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj);
--i;
}
@@ -4283,11 +2973,322 @@
}
}
-void MemoryGraphFixer::remove(Node* n) {
- assert(n->Opcode() == Op_ShenandoahWBMemProj, "");
- Node* c = _phase->get_ctrl(n);
- Node* mem = find_mem(c, NULL);
- if (mem == n) {
- _memory_nodes.map(c->_idx, mem->in(ShenandoahWBMemProjNode::WriteBarrier)->in(ShenandoahBarrierNode::Memory));
+ShenandoahLoadReferenceBarrierNode::ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* obj)
+: Node(ctrl, obj) {
+ ShenandoahBarrierSetC2::bsc2()->state()->add_load_reference_barrier(this);
+}
+
+const Type* ShenandoahLoadReferenceBarrierNode::bottom_type() const {
+ if (in(ValueIn) == NULL || in(ValueIn)->is_top()) {
+ return Type::TOP;
+ }
+ const Type* t = in(ValueIn)->bottom_type();
+ if (t == TypePtr::NULL_PTR) {
+ return t;
+ }
+ return t->is_oopptr();
+}
+
+const Type* ShenandoahLoadReferenceBarrierNode::Value(PhaseGVN* phase) const {
+ // Either input is TOP ==> the result is TOP
+ const Type *t2 = phase->type(in(ValueIn));
+ if( t2 == Type::TOP ) return Type::TOP;
+
+ if (t2 == TypePtr::NULL_PTR) {
+ return t2;
+ }
+
+ const Type* type = t2->is_oopptr()/*->cast_to_nonconst()*/;
+ return type;
+}
+
+Node* ShenandoahLoadReferenceBarrierNode::Identity(PhaseGVN* phase) {
+ Node* value = in(ValueIn);
+ if (!needs_barrier(phase, value)) {
+ return value;
+ }
+ return this;
+}
+
+bool ShenandoahLoadReferenceBarrierNode::needs_barrier(PhaseGVN* phase, Node* n) {
+ Unique_Node_List visited;
+ return needs_barrier_impl(phase, n, visited);
+}
+
+bool ShenandoahLoadReferenceBarrierNode::needs_barrier_impl(PhaseGVN* phase, Node* n, Unique_Node_List &visited) {
+ if (n == NULL) return false;
+ if (visited.member(n)) {
+ return false; // Been there.
+ }
+ visited.push(n);
+
+ if (n->is_Allocate()) {
+ // tty->print_cr("optimize barrier on alloc");
+ return false;
+ }
+ if (n->is_Call()) {
+ // tty->print_cr("optimize barrier on call");
+ return false;
+ }
+
+ const Type* type = phase->type(n);
+ if (type == Type::TOP) {
+ return false;
+ }
+ if (type->make_ptr()->higher_equal(TypePtr::NULL_PTR)) {
+ // tty->print_cr("optimize barrier on null");
+ return false;
+ }
+ if (type->make_oopptr() && type->make_oopptr()->const_oop() != NULL) {
+ // tty->print_cr("optimize barrier on constant");
+ return false;
+ }
+
+ switch (n->Opcode()) {
+ case Op_AddP:
+ return true; // TODO: Can refine?
+ case Op_LoadP:
+ case Op_ShenandoahCompareAndExchangeN:
+ case Op_ShenandoahCompareAndExchangeP:
+ case Op_CompareAndExchangeN:
+ case Op_CompareAndExchangeP:
+ case Op_GetAndSetN:
+ case Op_GetAndSetP:
+ return true;
+ case Op_Phi: {
+ for (uint i = 1; i < n->req(); i++) {
+ if (needs_barrier_impl(phase, n->in(i), visited)) return true;
+ }
+ return false;
+ }
+ case Op_CheckCastPP:
+ case Op_CastPP:
+ return needs_barrier_impl(phase, n->in(1), visited);
+ case Op_Proj:
+ return needs_barrier_impl(phase, n->in(0), visited);
+ case Op_ShenandoahLoadReferenceBarrier:
+ // tty->print_cr("optimize barrier on barrier");
+ return false;
+ case Op_Parm:
+ // tty->print_cr("optimize barrier on input arg");
+ return false;
+ case Op_DecodeN:
+ case Op_EncodeP:
+ return needs_barrier_impl(phase, n->in(1), visited);
+ case Op_LoadN:
+ return true;
+ case Op_CMoveP:
+ return needs_barrier_impl(phase, n->in(2), visited) ||
+ needs_barrier_impl(phase, n->in(3), visited);
+ case Op_ShenandoahEnqueueBarrier:
+ return needs_barrier_impl(phase, n->in(1), visited);
+ default:
+ break;
}
+#ifdef ASSERT
+ tty->print("need barrier on?: ");
+ tty->print_cr("ins:");
+ n->dump(2);
+ tty->print_cr("outs:");
+ n->dump(-2);
+ ShouldNotReachHere();
+#endif
+ return true;
}
+
+ShenandoahLoadReferenceBarrierNode::Strength ShenandoahLoadReferenceBarrierNode::get_barrier_strength() {
+ Unique_Node_List visited;
+ Node_Stack stack(0);
+ stack.push(this, 0);
+ Strength strength = NONE;
+ while (strength != STRONG && stack.size() > 0) {
+ Node* n = stack.node();
+ if (visited.member(n)) {
+ stack.pop();
+ continue;
+ }
+ visited.push(n);
+ bool visit_users = false;
+ switch (n->Opcode()) {
+ case Op_StoreN:
+ case Op_StoreP: {
+ strength = STRONG;
+ break;
+ }
+ case Op_CmpP: {
+ if (!n->in(1)->bottom_type()->higher_equal(TypePtr::NULL_PTR) &&
+ !n->in(2)->bottom_type()->higher_equal(TypePtr::NULL_PTR)) {
+ strength = STRONG;
+ }
+ break;
+ }
+ case Op_CallStaticJava: {
+ strength = STRONG;
+ break;
+ }
+ case Op_CallDynamicJava:
+ case Op_CallLeaf:
+ case Op_CallLeafNoFP:
+ case Op_CompareAndSwapL:
+ case Op_CompareAndSwapI:
+ case Op_CompareAndSwapB:
+ case Op_CompareAndSwapS:
+ case Op_ShenandoahCompareAndSwapN:
+ case Op_ShenandoahCompareAndSwapP:
+ case Op_ShenandoahWeakCompareAndSwapN:
+ case Op_ShenandoahWeakCompareAndSwapP:
+ case Op_ShenandoahCompareAndExchangeN:
+ case Op_ShenandoahCompareAndExchangeP:
+ case Op_CompareAndExchangeL:
+ case Op_CompareAndExchangeI:
+ case Op_CompareAndExchangeB:
+ case Op_CompareAndExchangeS:
+ case Op_WeakCompareAndSwapL:
+ case Op_WeakCompareAndSwapI:
+ case Op_WeakCompareAndSwapB:
+ case Op_WeakCompareAndSwapS:
+ case Op_GetAndSetL:
+ case Op_GetAndSetI:
+ case Op_GetAndSetB:
+ case Op_GetAndSetS:
+ case Op_GetAndSetP:
+ case Op_GetAndSetN:
+ case Op_GetAndAddL:
+ case Op_GetAndAddI:
+ case Op_GetAndAddB:
+ case Op_GetAndAddS:
+ case Op_ShenandoahEnqueueBarrier:
+ case Op_FastLock:
+ case Op_FastUnlock:
+ case Op_Rethrow:
+ case Op_Return:
+ case Op_StoreB:
+ case Op_StoreC:
+ case Op_StoreD:
+ case Op_StoreF:
+ case Op_StoreL:
+ case Op_StoreLConditional:
+ case Op_StoreI:
+ case Op_StoreVector:
+ case Op_StrInflatedCopy:
+ case Op_StrCompressedCopy:
+ case Op_EncodeP:
+ case Op_CastP2X:
+ case Op_SafePoint:
+ case Op_EncodeISOArray:
+ strength = STRONG;
+ break;
+ case Op_LoadB:
+ case Op_LoadUB:
+ case Op_LoadUS:
+ case Op_LoadD:
+ case Op_LoadF:
+ case Op_LoadL:
+ case Op_LoadI:
+ case Op_LoadS:
+ case Op_LoadN:
+ case Op_LoadP:
+ case Op_LoadVector: {
+ const TypePtr* adr_type = n->adr_type();
+ int alias_idx = Compile::current()->get_alias_index(adr_type);
+ Compile::AliasType* alias_type = Compile::current()->alias_type(alias_idx);
+ ciField* field = alias_type->field();
+ bool is_static = field != NULL && field->is_static();
+ bool is_final = field != NULL && field->is_final();
+ bool is_stable = field != NULL && field->is_stable();
+ if (ShenandoahOptimizeStaticFinals && is_static && is_final) {
+ // Leave strength as is.
+ } else if (ShenandoahOptimizeInstanceFinals && !is_static && is_final) {
+ // Leave strength as is.
+ } else if (ShenandoahOptimizeStableFinals && (is_stable || (adr_type->isa_aryptr() && adr_type->isa_aryptr()->is_stable()))) {
+ // Leave strength as is.
+ } else {
+ strength = WEAK;
+ }
+ break;
+ }
+ case Op_AryEq: {
+ Node* n1 = n->in(2);
+ Node* n2 = n->in(3);
+ if (!ShenandoahOptimizeStableFinals ||
+ !n1->bottom_type()->isa_aryptr() || !n1->bottom_type()->isa_aryptr()->is_stable() ||
+ !n2->bottom_type()->isa_aryptr() || !n2->bottom_type()->isa_aryptr()->is_stable()) {
+ strength = WEAK;
+ }
+ break;
+ }
+ case Op_StrEquals:
+ case Op_StrComp:
+ case Op_StrIndexOf:
+ case Op_StrIndexOfChar:
+ if (!ShenandoahOptimizeStableFinals) {
+ strength = WEAK;
+ }
+ break;
+ case Op_Conv2B:
+ case Op_LoadRange:
+ case Op_LoadKlass:
+ case Op_LoadNKlass:
+ // NONE, i.e. leave current strength as is
+ break;
+ case Op_AddP:
+ case Op_CheckCastPP:
+ case Op_CastPP:
+ case Op_CMoveP:
+ case Op_Phi:
+ case Op_ShenandoahLoadReferenceBarrier:
+ visit_users = true;
+ break;
+ default: {
+#ifdef ASSERT
+ tty->print_cr("Unknown node in get_barrier_strength:");
+ n->dump(1);
+ ShouldNotReachHere();
+#else
+ strength = STRONG;
+#endif
+ }
+ }
+#ifdef ASSERT
+/*
+ if (strength == STRONG) {
+ tty->print("strengthening node: ");
+ n->dump();
+ }
+ */
+#endif
+ stack.pop();
+ if (visit_users) {
+ for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
+ Node* user = n->fast_out(i);
+ if (user != NULL) {
+ stack.push(user, 0);
+ }
+ }
+ }
+ }
+ return strength;
+}
+
+CallStaticJavaNode* ShenandoahLoadReferenceBarrierNode::pin_and_expand_null_check(PhaseIterGVN& igvn) {
+ Node* val = in(ValueIn);
+
+ const Type* val_t = igvn.type(val);
+
+ if (val_t->meet(TypePtr::NULL_PTR) != val_t &&
+ val->Opcode() == Op_CastPP &&
+ val->in(0) != NULL &&
+ val->in(0)->Opcode() == Op_IfTrue &&
+ val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) &&
+ val->in(0)->in(0)->is_If() &&
+ val->in(0)->in(0)->in(1)->Opcode() == Op_Bool &&
+ val->in(0)->in(0)->in(1)->as_Bool()->_test._test == BoolTest::ne &&
+ val->in(0)->in(0)->in(1)->in(1)->Opcode() == Op_CmpP &&
+ val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1) &&
+ val->in(0)->in(0)->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) {
+ assert(val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1), "");
+ CallStaticJavaNode* unc = val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
+ return unc;
+ }
+ return NULL;
+}
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp Tue Apr 02 23:00:22 2019 +0200
@@ -36,10 +36,8 @@
class PhaseGVN;
class MemoryGraphFixer;
-class ShenandoahBarrierNode : public TypeNode {
+class ShenandoahBarrierC2Support : public AllStatic {
private:
- bool _allow_fromspace;
-
#ifdef ASSERT
enum verify_type {
ShenandoahLoad,
@@ -50,204 +48,49 @@
};
static bool verify_helper(Node* in, Node_Stack& phis, VectorSet& visited, verify_type t, bool trace, Unique_Node_List& barriers_used);
-#endif
-
-public:
- enum { Control,
- Memory,
- ValueIn
- };
-
- ShenandoahBarrierNode(Node* ctrl, Node* mem, Node* obj, bool allow_fromspace)
- : TypeNode(obj->bottom_type()->isa_oopptr() ? obj->bottom_type()->is_oopptr()->cast_to_nonconst() : obj->bottom_type(), 3),
- _allow_fromspace(allow_fromspace) {
-
- init_req(Control, ctrl);
- init_req(Memory, mem);
- init_req(ValueIn, obj);
-
- init_class_id(Class_ShenandoahBarrier);
- }
-
- static Node* skip_through_barrier(Node* n);
-
- static const TypeOopPtr* brooks_pointer_type(const Type* t) {
- return t->is_oopptr()->cast_to_nonconst()->add_offset(ShenandoahBrooksPointer::byte_offset())->is_oopptr();
- }
-
- virtual const TypePtr* adr_type() const {
- if (bottom_type() == Type::TOP) {
- return NULL;
- }
- //const TypePtr* adr_type = in(MemNode::Address)->bottom_type()->is_ptr();
- const TypePtr* adr_type = brooks_pointer_type(bottom_type());
- assert(adr_type->offset() == ShenandoahBrooksPointer::byte_offset(), "sane offset");
- assert(Compile::current()->alias_type(adr_type)->is_rewritable(), "brooks ptr must be rewritable");
- return adr_type;
- }
-
- virtual uint ideal_reg() const { return Op_RegP; }
- virtual uint match_edge(uint idx) const {
- return idx >= ValueIn;
- }
-
- Node* Identity_impl(PhaseGVN* phase);
-
- virtual const Type* Value(PhaseGVN* phase) const;
- virtual bool depends_only_on_test() const {
- return true;
- };
-
- static bool needs_barrier(PhaseGVN* phase, ShenandoahBarrierNode* orig, Node* n, Node* rb_mem, bool allow_fromspace);
-
-#ifdef ASSERT
static void report_verify_failure(const char* msg, Node* n1 = NULL, Node* n2 = NULL);
- static void verify(RootNode* root);
static void verify_raw_mem(RootNode* root);
#endif
-#ifndef PRODUCT
- virtual void dump_spec(outputStream *st) const;
-#endif
-
- // protected:
- static Node* dom_mem(Node* mem, Node*& mem_ctrl, Node* n, Node* rep_ctrl, int alias, PhaseIdealLoop* phase);
static Node* dom_mem(Node* mem, Node* ctrl, int alias, Node*& mem_ctrl, PhaseIdealLoop* phase);
- static bool is_dominator(Node *d_c, Node *n_c, Node* d, Node* n, PhaseIdealLoop* phase);
- static bool is_dominator_same_ctrl(Node* c, Node* d, Node* n, PhaseIdealLoop* phase);
static Node* no_branches(Node* c, Node* dom, bool allow_one_proj, PhaseIdealLoop* phase);
- static bool build_loop_late_post(PhaseIdealLoop* phase, Node* n);
- bool sink_node(PhaseIdealLoop* phase, Node* ctrl, Node* n_ctrl);
-
-protected:
- uint hash() const;
- bool cmp(const Node& n) const;
- uint size_of() const;
-
-private:
- static bool needs_barrier_impl(PhaseGVN* phase, ShenandoahBarrierNode* orig, Node* n, Node* rb_mem, bool allow_fromspace, Unique_Node_List &visited);
-
- static bool dominates_memory(PhaseGVN* phase, Node* b1, Node* b2, bool linear);
- static bool dominates_memory_impl(PhaseGVN* phase, Node* b1, Node* b2, Node* current, bool linear);
-};
-
-class ShenandoahReadBarrierNode : public ShenandoahBarrierNode {
-public:
- ShenandoahReadBarrierNode(Node* ctrl, Node* mem, Node* obj)
- : ShenandoahBarrierNode(ctrl, mem, obj, true) {
- assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier ||
- ShenandoahWriteBarrier || ShenandoahAcmpBarrier),
- "should be enabled");
- }
- ShenandoahReadBarrierNode(Node* ctrl, Node* mem, Node* obj, bool allow_fromspace)
- : ShenandoahBarrierNode(ctrl, mem, obj, allow_fromspace) {
- assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier ||
- ShenandoahWriteBarrier || ShenandoahAcmpBarrier),
- "should be enabled");
- }
-
- virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
- virtual Node* Identity(PhaseGVN* phase);
- virtual int Opcode() const;
-
- bool is_independent(Node* mem);
-
- void try_move(PhaseIdealLoop* phase);
-
-private:
- static bool is_independent(const Type* in_type, const Type* this_type);
- static bool dominates_memory_rb(PhaseGVN* phase, Node* b1, Node* b2, bool linear);
- static bool dominates_memory_rb_impl(PhaseGVN* phase, Node* b1, Node* b2, Node* current, bool linear);
-};
-
-class ShenandoahWriteBarrierNode : public ShenandoahBarrierNode {
-public:
- ShenandoahWriteBarrierNode(Compile* C, Node* ctrl, Node* mem, Node* obj);
-
- virtual int Opcode() const;
- virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
- virtual Node* Identity(PhaseGVN* phase);
- virtual bool depends_only_on_test() const { return false; }
-
- static bool expand(Compile* C, PhaseIterGVN& igvn);
- static bool is_gc_state_load(Node *n);
static bool is_heap_state_test(Node* iff, int mask);
- static bool is_heap_stable_test(Node* iff);
static bool try_common_gc_state_load(Node *n, PhaseIdealLoop *phase);
static bool has_safepoint_between(Node* start, Node* stop, PhaseIdealLoop *phase);
-
- static LoopNode* try_move_before_pre_loop(Node* c, Node* val_ctrl, PhaseIdealLoop* phase);
- static Node* move_above_predicates(LoopNode* cl, Node* val_ctrl, PhaseIdealLoop* phase);
-#ifdef ASSERT
- static bool memory_dominates_all_paths(Node* mem, Node* rep_ctrl, int alias, PhaseIdealLoop* phase);
- static void memory_dominates_all_paths_helper(Node* c, Node* rep_ctrl, Unique_Node_List& controls, PhaseIdealLoop* phase);
-#endif
- void try_move_before_loop(GrowableArray<MemoryGraphFixer*>& memory_graph_fixers, PhaseIdealLoop* phase, bool include_lsm, Unique_Node_List& uses);
- void try_move_before_loop_helper(LoopNode* cl, Node* val_ctrl, GrowableArray<MemoryGraphFixer*>& memory_graph_fixers, PhaseIdealLoop* phase, bool include_lsm, Unique_Node_List& uses);
- static void pin_and_expand(PhaseIdealLoop* phase);
- CallStaticJavaNode* pin_and_expand_null_check(PhaseIterGVN& igvn);
- void pin_and_expand_move_barrier(PhaseIdealLoop* phase, GrowableArray<MemoryGraphFixer*>& memory_graph_fixers, Unique_Node_List& uses);
- void pin_and_expand_helper(PhaseIdealLoop* phase);
static Node* find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase);
static void follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase);
static void test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase);
-
static void test_heap_stable(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl,
PhaseIdealLoop* phase);
- static void call_wb_stub(Node*& ctrl, Node*& val, Node*& result_mem,
- Node* raw_mem, Node* wb_mem, int alias,
- PhaseIdealLoop* phase);
+ static void call_lrb_stub(Node*& ctrl, Node*& val, Node*& result_mem, Node* raw_mem, PhaseIdealLoop* phase);
static Node* clone_null_check(Node*& c, Node* val, Node* unc_ctrl, PhaseIdealLoop* phase);
static void fix_null_check(Node* unc, Node* unc_ctrl, Node* new_unc_ctrl, Unique_Node_List& uses,
PhaseIdealLoop* phase);
static void in_cset_fast_test(Node*& ctrl, Node*& not_cset_ctrl, Node* val, Node* raw_mem, PhaseIdealLoop* phase);
static void move_heap_stable_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase);
-
- static void optimize_after_expansion(VectorSet &visited, Node_Stack &nstack, Node_List &old_new, PhaseIdealLoop* phase);
static void merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase);
static bool identical_backtoback_ifs(Node *n, PhaseIdealLoop* phase);
static void fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase);
-
- static void optimize_before_expansion(PhaseIdealLoop* phase, GrowableArray<MemoryGraphFixer*> memory_graph_fixers, bool include_lsm);
- Node* would_subsume(ShenandoahBarrierNode* other, PhaseIdealLoop* phase);
static IfNode* find_unswitching_candidate(const IdealLoopTree *loop, PhaseIdealLoop* phase);
- Node* try_split_thru_phi(PhaseIdealLoop* phase);
-};
+public:
+ static bool is_dominator(Node* d_c, Node* n_c, Node* d, Node* n, PhaseIdealLoop* phase);
+ static bool is_dominator_same_ctrl(Node* c, Node* d, Node* n, PhaseIdealLoop* phase);
-class ShenandoahWBMemProjNode : public Node {
-public:
- enum { Control,
- WriteBarrier };
-
- ShenandoahWBMemProjNode(Node *src) : Node(NULL, src) {
- assert(UseShenandoahGC && ShenandoahWriteBarrier, "should be enabled");
- assert(src->Opcode() == Op_ShenandoahWriteBarrier || src->is_Mach(), "epxect wb");
- }
- virtual Node* Identity(PhaseGVN* phase);
+ static bool is_gc_state_load(Node* n);
+ static bool is_heap_stable_test(Node* iff);
- virtual int Opcode() const;
- virtual bool is_CFG() const { return false; }
- virtual const Type *bottom_type() const {return Type::MEMORY;}
- virtual const TypePtr *adr_type() const {
- Node* wb = in(WriteBarrier);
- if (wb == NULL || wb->is_top()) return NULL; // node is dead
- assert(wb->Opcode() == Op_ShenandoahWriteBarrier || (wb->is_Mach() && wb->as_Mach()->ideal_Opcode() == Op_ShenandoahWriteBarrier) || wb->is_Phi(), "expect wb");
- return ShenandoahBarrierNode::brooks_pointer_type(wb->bottom_type());
- }
+ static bool expand(Compile* C, PhaseIterGVN& igvn);
+ static void pin_and_expand(PhaseIdealLoop* phase);
+ static void optimize_after_expansion(VectorSet& visited, Node_Stack& nstack, Node_List& old_new, PhaseIdealLoop* phase);
- virtual uint ideal_reg() const { return 0;} // memory projections don't have a register
- virtual const Type *Value(PhaseGVN* phase ) const {
- return bottom_type();
- }
-#ifndef PRODUCT
- virtual void dump_spec(outputStream *st) const {};
+#ifdef ASSERT
+ static void verify(RootNode* root);
#endif
};
class ShenandoahEnqueueBarrierNode : public Node {
public:
- ShenandoahEnqueueBarrierNode(Node* val) : Node(NULL, val) {
- }
+ ShenandoahEnqueueBarrierNode(Node* val);
const Type *bottom_type() const;
const Type* Value(PhaseGVN* phase) const;
@@ -289,7 +132,6 @@
Node* find_mem(Node* ctrl, Node* n) const;
void fix_mem(Node* ctrl, Node* region, Node* mem, Node* mem_for_ctrl, Node* mem_phi, Unique_Node_List& uses);
int alias() const { return _alias; }
- void remove(Node* n);
};
class ShenandoahCompareAndSwapPNode : public CompareAndSwapPNode {
@@ -382,4 +224,41 @@
virtual int Opcode() const;
};
+class ShenandoahLoadReferenceBarrierNode : public Node {
+public:
+ enum {
+ Control,
+ ValueIn
+ };
+
+ enum Strength {
+ NONE, WEAK, STRONG, NA
+ };
+
+ ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* val);
+
+ virtual int Opcode() const;
+ virtual const Type* bottom_type() const;
+ virtual const Type* Value(PhaseGVN* phase) const;
+ virtual const class TypePtr *adr_type() const { return TypeOopPtr::BOTTOM; }
+ virtual uint match_edge(uint idx) const {
+ return idx >= ValueIn;
+ }
+ virtual uint ideal_reg() const { return Op_RegP; }
+
+ virtual Node* Identity(PhaseGVN* phase);
+
+ uint size_of() const {
+ return sizeof(*this);
+ }
+
+ Strength get_barrier_strength();
+ CallStaticJavaNode* pin_and_expand_null_check(PhaseIterGVN& igvn);
+
+private:
+ bool needs_barrier(PhaseGVN* phase, Node* n);
+ bool needs_barrier_impl(PhaseGVN* phase, Node* n, Unique_Node_List &visited);
+};
+
+
#endif // SHARE_GC_SHENANDOAH_C2_SHENANDOAHSUPPORT_HPP
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp Tue Apr 02 23:00:22 2019 +0200
@@ -41,13 +41,10 @@
SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahImplicitGCInvokesConcurrent);
// Final configuration checks
+ SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier);
- SHENANDOAH_CHECK_FLAG_SET(ShenandoahReadBarrier);
- SHENANDOAH_CHECK_FLAG_SET(ShenandoahWriteBarrier);
- SHENANDOAH_CHECK_FLAG_SET(ShenandoahStoreValReadBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahKeepAliveBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier);
- SHENANDOAH_CHECK_FLAG_SET(ShenandoahAcmpBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier);
}
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp Tue Apr 02 23:00:22 2019 +0200
@@ -47,13 +47,10 @@
}
// Final configuration checks
+ SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier);
- SHENANDOAH_CHECK_FLAG_SET(ShenandoahReadBarrier);
- SHENANDOAH_CHECK_FLAG_SET(ShenandoahWriteBarrier);
- SHENANDOAH_CHECK_FLAG_SET(ShenandoahStoreValReadBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahKeepAliveBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier);
- SHENANDOAH_CHECK_FLAG_SET(ShenandoahAcmpBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier);
}
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp Tue Apr 02 23:00:22 2019 +0200
@@ -42,13 +42,10 @@
SHENANDOAH_ERGO_OVERRIDE_DEFAULT(ShenandoahGarbageThreshold, 10);
// Final configuration checks
+ SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier);
- SHENANDOAH_CHECK_FLAG_SET(ShenandoahReadBarrier);
- SHENANDOAH_CHECK_FLAG_SET(ShenandoahWriteBarrier);
- SHENANDOAH_CHECK_FLAG_SET(ShenandoahStoreValReadBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahKeepAliveBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier);
- SHENANDOAH_CHECK_FLAG_SET(ShenandoahAcmpBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier);
}
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp Tue Apr 02 23:00:22 2019 +0200
@@ -43,14 +43,11 @@
}
// Disable known barriers by default.
+ SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahLoadRefBarrier);
SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahSATBBarrier);
SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahKeepAliveBarrier);
- SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahWriteBarrier);
- SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahReadBarrier);
SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahStoreValEnqueueBarrier);
- SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahStoreValReadBarrier);
SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahCASBarrier);
- SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahAcmpBarrier);
SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahCloneBarrier);
// Final configuration checks
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp Tue Apr 02 23:00:22 2019 +0200
@@ -40,13 +40,10 @@
SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahImplicitGCInvokesConcurrent);
// Final configuration checks
+ SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier);
- SHENANDOAH_CHECK_FLAG_SET(ShenandoahReadBarrier);
- SHENANDOAH_CHECK_FLAG_SET(ShenandoahWriteBarrier);
- SHENANDOAH_CHECK_FLAG_SET(ShenandoahStoreValReadBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahKeepAliveBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier);
- SHENANDOAH_CHECK_FLAG_SET(ShenandoahAcmpBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier);
}
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahTraversalHeuristics.cpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahTraversalHeuristics.cpp Tue Apr 02 23:00:22 2019 +0200
@@ -37,7 +37,6 @@
_last_cset_select(0)
{
FLAG_SET_DEFAULT(ShenandoahSATBBarrier, false);
- FLAG_SET_DEFAULT(ShenandoahStoreValReadBarrier, false);
FLAG_SET_DEFAULT(ShenandoahStoreValEnqueueBarrier, true);
FLAG_SET_DEFAULT(ShenandoahKeepAliveBarrier, false);
FLAG_SET_DEFAULT(ShenandoahAllowMixedAllocs, false);
@@ -53,11 +52,9 @@
SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahImplicitGCInvokesConcurrent);
// Final configuration checks
- SHENANDOAH_CHECK_FLAG_SET(ShenandoahReadBarrier);
- SHENANDOAH_CHECK_FLAG_SET(ShenandoahWriteBarrier);
+ SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahStoreValEnqueueBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier);
- SHENANDOAH_CHECK_FLAG_SET(ShenandoahAcmpBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier);
}
--- a/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp Tue Apr 02 23:00:22 2019 +0200
@@ -46,12 +46,8 @@
FLAG_SET_DEFAULT(ShenandoahSATBBarrier, false);
FLAG_SET_DEFAULT(ShenandoahKeepAliveBarrier, false);
- FLAG_SET_DEFAULT(ShenandoahWriteBarrier, false);
- FLAG_SET_DEFAULT(ShenandoahReadBarrier, false);
FLAG_SET_DEFAULT(ShenandoahStoreValEnqueueBarrier, false);
- FLAG_SET_DEFAULT(ShenandoahStoreValReadBarrier, false);
FLAG_SET_DEFAULT(ShenandoahCASBarrier, false);
- FLAG_SET_DEFAULT(ShenandoahAcmpBarrier, false);
FLAG_SET_DEFAULT(ShenandoahCloneBarrier, false);
#endif
@@ -111,12 +107,8 @@
if (ShenandoahVerifyOptoBarriers &&
(!FLAG_IS_DEFAULT(ShenandoahSATBBarrier) ||
!FLAG_IS_DEFAULT(ShenandoahKeepAliveBarrier) ||
- !FLAG_IS_DEFAULT(ShenandoahWriteBarrier) ||
- !FLAG_IS_DEFAULT(ShenandoahReadBarrier) ||
!FLAG_IS_DEFAULT(ShenandoahStoreValEnqueueBarrier) ||
- !FLAG_IS_DEFAULT(ShenandoahStoreValReadBarrier) ||
!FLAG_IS_DEFAULT(ShenandoahCASBarrier) ||
- !FLAG_IS_DEFAULT(ShenandoahAcmpBarrier) ||
!FLAG_IS_DEFAULT(ShenandoahCloneBarrier)
)) {
warning("Unusual barrier configuration, disabling C2 barrier verification");
@@ -164,13 +156,6 @@
FLAG_SET_DEFAULT(UseAOT, false);
}
- // JNI fast get field stuff is not currently supported by Shenandoah.
- // It would introduce another heap memory access for reading the forwarding
- // pointer, which would have to be guarded by the signal handler machinery.
- // See:
- // http://mail.openjdk.java.net/pipermail/hotspot-dev/2018-June/032763.html
- FLAG_SET_DEFAULT(UseFastJNIAccessors, false);
-
// TLAB sizing policy makes resizing decisions before each GC cycle. It averages
// historical data, assigning more recent data the weight according to TLABAllocationWeight.
// Current default is good for generational collectors that run frequent young GCs.
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp Tue Apr 02 23:00:22 2019 +0200
@@ -218,31 +218,25 @@
}
}
-oop ShenandoahBarrierSet::read_barrier(oop src) {
- // Check for forwarded objects, because on Full GC path we might deal with
- // non-trivial fwdptrs that contain Full GC specific metadata. We could check
- // for is_full_gc_in_progress(), but this also covers the case of stable heap,
- // which provides a bit of performance improvement.
- if (ShenandoahReadBarrier && _heap->has_forwarded_objects()) {
- return ShenandoahBarrierSet::resolve_forwarded(src);
+oop ShenandoahBarrierSet::load_reference_barrier_not_null(oop obj) {
+ if (ShenandoahLoadRefBarrier && _heap->has_forwarded_objects()) {
+ return load_reference_barrier_impl(obj);
} else {
- return src;
+ return obj;
}
}
-bool ShenandoahBarrierSet::obj_equals(oop obj1, oop obj2) {
- bool eq = oopDesc::equals_raw(obj1, obj2);
- if (! eq && ShenandoahAcmpBarrier) {
- OrderAccess::loadload();
- obj1 = resolve_forwarded(obj1);
- obj2 = resolve_forwarded(obj2);
- eq = oopDesc::equals_raw(obj1, obj2);
+oop ShenandoahBarrierSet::load_reference_barrier(oop obj) {
+ if (obj != NULL) {
+ return load_reference_barrier_not_null(obj);
+ } else {
+ return obj;
}
- return eq;
}
-oop ShenandoahBarrierSet::write_barrier_mutator(oop obj) {
- assert(UseShenandoahGC && ShenandoahWriteBarrier, "should be enabled");
+
+oop ShenandoahBarrierSet::load_reference_barrier_mutator(oop obj) {
+ assert(ShenandoahLoadRefBarrier, "should be enabled");
assert(_heap->is_gc_in_progress_mask(ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL), "evac should be in progress");
shenandoah_assert_in_cset(NULL, obj);
@@ -288,8 +282,8 @@
return fwd;
}
-oop ShenandoahBarrierSet::write_barrier_impl(oop obj) {
- assert(UseShenandoahGC && ShenandoahWriteBarrier, "should be enabled");
+oop ShenandoahBarrierSet::load_reference_barrier_impl(oop obj) {
+ assert(ShenandoahLoadRefBarrier, "should be enabled");
if (!CompressedOops::is_null(obj)) {
bool evac_in_progress = _heap->is_gc_in_progress_mask(ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
oop fwd = resolve_forwarded_not_null(obj);
@@ -311,23 +305,10 @@
}
}
-oop ShenandoahBarrierSet::write_barrier(oop obj) {
- if (ShenandoahWriteBarrier && _heap->has_forwarded_objects()) {
- return write_barrier_impl(obj);
- } else {
- return obj;
- }
-}
-
-oop ShenandoahBarrierSet::storeval_barrier(oop obj) {
+void ShenandoahBarrierSet::storeval_barrier(oop obj) {
if (ShenandoahStoreValEnqueueBarrier && !CompressedOops::is_null(obj) && _heap->is_concurrent_traversal_in_progress()) {
- obj = write_barrier(obj);
enqueue(obj);
}
- if (ShenandoahStoreValReadBarrier) {
- obj = resolve_forwarded(obj);
- }
- return obj;
}
void ShenandoahBarrierSet::keep_alive_barrier(oop obj) {
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp Tue Apr 02 23:00:22 2019 +0200
@@ -87,24 +87,15 @@
virtual void on_thread_attach(Thread* thread);
virtual void on_thread_detach(Thread* thread);
- virtual oop read_barrier(oop src);
-
static inline oop resolve_forwarded_not_null(oop p);
static inline oop resolve_forwarded(oop p);
- virtual oop write_barrier(oop obj);
-
- oop write_barrier_mutator(oop obj);
-
- virtual oop storeval_barrier(oop obj);
+ void storeval_barrier(oop obj);
+ void keep_alive_barrier(oop obj);
- virtual void keep_alive_barrier(oop obj);
-
- bool obj_equals(oop obj1, oop obj2);
-
-#ifdef CHECK_UNHANDLED_OOPS
- bool oop_equals_operator_allowed() { return !ShenandoahVerifyObjectEquals; }
-#endif
+ oop load_reference_barrier(oop obj);
+ oop load_reference_barrier_mutator(oop obj);
+ oop load_reference_barrier_not_null(oop obj);
void enqueue(oop obj);
@@ -114,7 +105,7 @@
template <class T, bool STOREVAL_WRITE_BARRIER>
void write_ref_array_loop(HeapWord* start, size_t count);
- oop write_barrier_impl(oop obj);
+ oop load_reference_barrier_impl(oop obj);
static void keep_alive_if_weak(DecoratorSet decorators, oop value) {
assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Reference strength must be known");
@@ -149,114 +140,31 @@
class AccessBarrier: public BarrierSet::AccessBarrier<decorators, BarrierSetT> {
typedef BarrierSet::AccessBarrier<decorators, BarrierSetT> Raw;
- public:
- // Primitive heap accesses. These accessors get resolved when
- // IN_HEAP is set (e.g. when using the HeapAccess API), it is
- // not an oop_* overload, and the barrier strength is AS_NORMAL.
template <typename T>
- static T load_in_heap(T* addr) {
- ShouldNotReachHere();
- return Raw::template load<T>(addr);
- }
-
- template <typename T>
- static T load_in_heap_at(oop base, ptrdiff_t offset) {
- base = ShenandoahBarrierSet::resolve_forwarded(base);
- return Raw::template load_at<T>(base, offset);
- }
-
- template <typename T>
- static void store_in_heap(T* addr, T value) {
- ShouldNotReachHere();
- Raw::store(addr, value);
- }
-
- template <typename T>
- static void store_in_heap_at(oop base, ptrdiff_t offset, T value) {
- base = ShenandoahBarrierSet::barrier_set()->write_barrier(base);
- Raw::store_at(base, offset, value);
- }
+ static oop oop_atomic_cmpxchg_in_heap_impl(oop new_value, T* addr, oop compare_value);
template <typename T>
- static T atomic_cmpxchg_in_heap(T new_value, T* addr, T compare_value) {
- ShouldNotReachHere();
- return Raw::atomic_cmpxchg(new_value, addr, compare_value);
- }
-
- template <typename T>
- static T atomic_cmpxchg_in_heap_at(T new_value, oop base, ptrdiff_t offset, T compare_value) {
- base = ShenandoahBarrierSet::barrier_set()->write_barrier(base);
- return Raw::atomic_cmpxchg_at(new_value, base, offset, compare_value);
- }
+ static oop oop_atomic_xchg_in_heap_impl(oop new_value, T* addr);
- template <typename T>
- static T atomic_xchg_in_heap(T new_value, T* addr) {
- ShouldNotReachHere();
- return Raw::atomic_xchg(new_value, addr);
- }
-
- template <typename T>
- static T atomic_xchg_in_heap_at(T new_value, oop base, ptrdiff_t offset) {
- base = ShenandoahBarrierSet::barrier_set()->write_barrier(base);
- return Raw::atomic_xchg_at(new_value, base, offset);
- }
-
- template <typename T>
- static void arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
- arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
- size_t length);
-
+ public:
// Heap oop accesses. These accessors get resolved when
// IN_HEAP is set (e.g. when using the HeapAccess API), it is
// an oop_* overload, and the barrier strength is AS_NORMAL.
template <typename T>
- static oop oop_load_in_heap(T* addr) {
- // ShouldNotReachHere();
- oop value = Raw::template oop_load<oop>(addr);
- keep_alive_if_weak(decorators, value);
- return value;
- }
-
- static oop oop_load_in_heap_at(oop base, ptrdiff_t offset) {
- base = ShenandoahBarrierSet::resolve_forwarded(base);
- oop value = Raw::template oop_load_at<oop>(base, offset);
- keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), value);
- return value;
- }
+ static oop oop_load_in_heap(T* addr);
+ static oop oop_load_in_heap_at(oop base, ptrdiff_t offset);
template <typename T>
- static void oop_store_in_heap(T* addr, oop value) {
- const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0;
- if (keep_alive) {
- ShenandoahBarrierSet::barrier_set()->write_ref_field_pre_work(addr, value);
- }
- Raw::oop_store(addr, value);
- }
-
- static void oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) {
- base = ShenandoahBarrierSet::barrier_set()->write_barrier(base);
- value = ShenandoahBarrierSet::barrier_set()->storeval_barrier(value);
-
- oop_store_in_heap(AccessInternal::oop_field_addr<decorators>(base, offset), value);
- }
+ static void oop_store_in_heap(T* addr, oop value);
+ static void oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value);
template <typename T>
static oop oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value);
-
- static oop oop_atomic_cmpxchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset, oop compare_value) {
- base = ShenandoahBarrierSet::barrier_set()->write_barrier(base);
- new_value = ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value);
- return oop_atomic_cmpxchg_in_heap(new_value, AccessInternal::oop_field_addr<decorators>(base, offset), compare_value);
- }
+ static oop oop_atomic_cmpxchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset, oop compare_value);
template <typename T>
static oop oop_atomic_xchg_in_heap(oop new_value, T* addr);
-
- static oop oop_atomic_xchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset) {
- base = ShenandoahBarrierSet::barrier_set()->write_barrier(base);
- new_value = ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value);
- return oop_atomic_xchg_in_heap(new_value, AccessInternal::oop_field_addr<decorators>(base, offset));
- }
+ static oop oop_atomic_xchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset);
template <typename T>
static bool oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
@@ -268,19 +176,13 @@
// Needed for loads on non-heap weak references
template <typename T>
- static oop oop_load_not_in_heap(T* addr) {
- oop value = Raw::oop_load_not_in_heap(addr);
- keep_alive_if_weak(decorators, value);
- return value;
- }
+ static oop oop_load_not_in_heap(T* addr);
- static oop resolve(oop obj) {
- return ShenandoahBarrierSet::barrier_set()->write_barrier(obj);
- }
+ template <typename T>
+ static oop oop_atomic_cmpxchg_not_in_heap(oop new_value, T* addr, oop compare_value);
- static bool equals(oop o1, oop o2) {
- return ShenandoahBarrierSet::barrier_set()->obj_equals(o1, o2);
- }
+ template <typename T>
+ static oop oop_atomic_xchg_not_in_heap(oop new_value, T* addr);
};
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp Tue Apr 02 23:00:22 2019 +0200
@@ -52,7 +52,49 @@
template <DecoratorSet decorators, typename BarrierSetT>
template <typename T>
-inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value) {
+inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap(T* addr) {
+ oop value = Raw::oop_load_in_heap(addr);
+ value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(value);
+ keep_alive_if_weak(decorators, value);
+ return value;
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap_at(oop base, ptrdiff_t offset) {
+ oop value = Raw::oop_load_in_heap_at(base, offset);
+ value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(value);
+ keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), value);
+ return value;
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+template <typename T>
+inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(T* addr) {
+ oop value = Raw::oop_load_not_in_heap(addr);
+ value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(value);
+ keep_alive_if_weak(decorators, value);
+ return value;
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+template <typename T>
+inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap(T* addr, oop value) {
+ ShenandoahBarrierSet::barrier_set()->storeval_barrier(value);
+ const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0;
+ if (keep_alive) {
+ ShenandoahBarrierSet::barrier_set()->write_ref_field_pre_work(addr, value);
+ }
+ Raw::oop_store_in_heap(addr, value);
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) {
+ oop_store_in_heap(AccessInternal::oop_field_addr<decorators>(base, offset), value);
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+template <typename T>
+inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_not_in_heap(oop new_value, T* addr, oop compare_value) {
oop res;
oop expected = compare_value;
do {
@@ -60,42 +102,79 @@
res = Raw::oop_atomic_cmpxchg(new_value, addr, compare_value);
expected = res;
} while ((! oopDesc::equals_raw(compare_value, expected)) && oopDesc::equals_raw(resolve_forwarded(compare_value), resolve_forwarded(expected)));
- if (oopDesc::equals_raw(expected, compare_value)) {
- const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0;
- if (keep_alive && ShenandoahSATBBarrier && !CompressedOops::is_null(compare_value) &&
- ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) {
- ShenandoahBarrierSet::barrier_set()->enqueue(compare_value);
- }
+ if (res != NULL) {
+ return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(res);
+ } else {
+ return res;
+ }
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+template <typename T>
+inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap_impl(oop new_value, T* addr, oop compare_value) {
+ ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value);
+ oop result = oop_atomic_cmpxchg_not_in_heap(new_value, addr, compare_value);
+ const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0;
+ if (keep_alive && ShenandoahSATBBarrier && !CompressedOops::is_null(result) &&
+ oopDesc::equals_raw(result, compare_value) &&
+ ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) {
+ ShenandoahBarrierSet::barrier_set()->enqueue(result);
}
- return res;
+ return result;
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+template <typename T>
+inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value) {
+ oop result = oop_atomic_cmpxchg_in_heap_impl(new_value, addr, compare_value);
+ keep_alive_if_weak(decorators, result);
+ return result;
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset, oop compare_value) {
+ oop result = oop_atomic_cmpxchg_in_heap_impl(new_value, AccessInternal::oop_field_addr<decorators>(base, offset), compare_value);
+ keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), result);
+ return result;
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+template <typename T>
+inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_not_in_heap(oop new_value, T* addr) {
+ oop previous = Raw::oop_atomic_xchg(new_value, addr);
+ if (previous != NULL) {
+ return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(previous);
+ } else {
+ return previous;
+ }
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+template <typename T>
+inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_impl(oop new_value, T* addr) {
+ ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value);
+ oop result = oop_atomic_xchg_not_in_heap(new_value, addr);
+ const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0;
+ if (keep_alive && ShenandoahSATBBarrier && !CompressedOops::is_null(result) &&
+ ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) {
+ ShenandoahBarrierSet::barrier_set()->enqueue(result);
+ }
+ return result;
}
template <DecoratorSet decorators, typename BarrierSetT>
template <typename T>
inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(oop new_value, T* addr) {
- oop previous = Raw::oop_atomic_xchg(new_value, addr);
- if (ShenandoahSATBBarrier) {
- const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0;
- if (keep_alive && !CompressedOops::is_null(previous) &&
- ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) {
- ShenandoahBarrierSet::barrier_set()->enqueue(previous);
- }
- }
- return previous;
+ oop result = oop_atomic_xchg_in_heap_impl(new_value, addr);
+ keep_alive_if_weak(addr, result);
+ return result;
}
template <DecoratorSet decorators, typename BarrierSetT>
-template <typename T>
-void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
- arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
- size_t length) {
- if (!CompressedOops::is_null(src_obj)) {
- src_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src_obj));
- }
- if (!CompressedOops::is_null(dst_obj)) {
- dst_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst_obj));
- }
- Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
+inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset) {
+ oop result = oop_atomic_xchg_in_heap_impl(new_value, AccessInternal::oop_field_addr<decorators>(base, offset));
+ keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), result);
+ return result;
}
template <typename T>
@@ -248,8 +327,6 @@
// Clone barrier support
template <DecoratorSet decorators, typename BarrierSetT>
void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {
- src = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src));
- dst = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst));
Raw::clone(src, dst, size);
ShenandoahBarrierSet::barrier_set()->write_region(MemRegion((HeapWord*) dst, size));
}
@@ -260,13 +337,6 @@
arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
size_t length) {
ShenandoahHeap* heap = ShenandoahHeap::heap();
- if (!CompressedOops::is_null(src_obj)) {
- src_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src_obj));
- }
- if (!CompressedOops::is_null(dst_obj)) {
- dst_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst_obj));
- }
-
bool satb = ShenandoahSATBBarrier && heap->is_concurrent_mark_in_progress();
bool checkcast = HasDecorator<decorators, ARRAYCOPY_CHECKCAST>::value;
bool disjoint = HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value;
--- a/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp Tue Apr 02 23:00:22 2019 +0200
@@ -119,39 +119,6 @@
}
};
-class ShenandoahNMethodOopInitializer : public OopClosure {
-private:
- ShenandoahHeap* const _heap;
-
-public:
- ShenandoahNMethodOopInitializer() : _heap(ShenandoahHeap::heap()) {};
-
-private:
- template <class T>
- inline void do_oop_work(T* p) {
- T o = RawAccess<>::oop_load(p);
- if (! CompressedOops::is_null(o)) {
- oop obj1 = CompressedOops::decode_not_null(o);
- oop obj2 = ShenandoahBarrierSet::barrier_set()->write_barrier(obj1);
- if (! oopDesc::equals_raw(obj1, obj2)) {
- shenandoah_assert_not_in_cset(NULL, obj2);
- RawAccess<IS_NOT_NULL>::oop_store(p, obj2);
- if (_heap->is_concurrent_traversal_in_progress()) {
- ShenandoahBarrierSet::barrier_set()->enqueue(obj2);
- }
- }
- }
- }
-
-public:
- void do_oop(oop* o) {
- do_oop_work(o);
- }
- void do_oop(narrowOop* o) {
- do_oop_work(o);
- }
-};
-
ShenandoahCodeRoots::PaddedLock ShenandoahCodeRoots::_recorded_nms_lock;
GrowableArray<ShenandoahNMethod*>* ShenandoahCodeRoots::_recorded_nms;
@@ -163,21 +130,13 @@
void ShenandoahCodeRoots::add_nmethod(nmethod* nm) {
switch (ShenandoahCodeRootsStyle) {
case 0:
- case 1: {
- ShenandoahNMethodOopInitializer init;
- nm->oops_do(&init);
- nm->fix_oop_relocations();
+ case 1:
break;
- }
case 2: {
ShenandoahNMethodOopDetector detector;
nm->oops_do(&detector);
if (detector.has_oops()) {
- ShenandoahNMethodOopInitializer init;
- nm->oops_do(&init);
- nm->fix_oop_relocations();
-
ShenandoahNMethod* nmr = new ShenandoahNMethod(nm, detector.oops());
nmr->assert_alive_and_correct();
--- a/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.hpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.hpp Tue Apr 02 23:00:22 2019 +0200
@@ -31,8 +31,8 @@
* Provides safe handling of out-of-memory situations during evacuation.
*
* When a Java thread encounters out-of-memory while evacuating an object in a
- * write-barrier (i.e. it cannot copy the object to to-space), it does not necessarily
- * follow we can return immediately from the WB (and store to from-space).
+ * load-reference-barrier (i.e. it cannot copy the object to to-space), it does not
+ * necessarily follow we can return immediately from the LRB (and store to from-space).
*
* In very basic case, on such failure we may wait until the the evacuation is over,
* and then resolve the forwarded copy, and to the store there. This is possible
@@ -64,17 +64,17 @@
* - failure:
* - if offending value is a valid counter, then try again
* - if offending value is OOM-during-evac special value: loop until
- * counter drops to 0, then exit with read-barrier
+ * counter drops to 0, then exit with resolving the ptr
*
* Upon exit, exiting thread will decrease the counter using atomic dec.
*
* Upon OOM-during-evac, any thread will attempt to CAS OOM-during-evac
* special value into the counter. Depending on result:
- * - success: busy-loop until counter drops to zero, then exit with RB
+ * - success: busy-loop until counter drops to zero, then exit with resolve
* - failure:
* - offender is valid counter update: try again
* - offender is OOM-during-evac: busy loop until counter drops to
- * zero, then exit with RB
+ * zero, then exit with resolve
*/
class ShenandoahEvacOOMHandler {
private:
@@ -94,7 +94,7 @@
*
* When this returns true, it is safe to continue with normal evacuation.
* When this method returns false, evacuation must not be entered, and caller
- * may safely continue with a read-barrier (if Java thread).
+ * may safely continue with a simple resolve (if Java thread).
*/
void enter_evacuation();
@@ -106,7 +106,7 @@
/**
* Signal out-of-memory during evacuation. It will prevent any other threads
* from entering the evacuation path, then wait until all threads have left the
- * evacuation path, and then return. It is then safe to continue with a read-barrier.
+ * evacuation path, and then return. It is then safe to continue with a simple resolve.
*/
void handle_out_of_memory_during_evacuation();
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp Tue Apr 02 23:00:22 2019 +0200
@@ -389,10 +389,6 @@
err_msg("Heuristics \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
_heuristics->name()));
}
-
- if (ShenandoahStoreValEnqueueBarrier && ShenandoahStoreValReadBarrier) {
- vm_exit_during_initialization("Cannot use both ShenandoahStoreValEnqueueBarrier and ShenandoahStoreValReadBarrier");
- }
log_info(gc, init)("Shenandoah heuristics: %s",
_heuristics->name());
} else {
@@ -791,7 +787,7 @@
assert(req.is_gc_alloc(), "Can only accept GC allocs here");
result = allocate_memory_under_lock(req, in_new_region);
// Do not call handle_alloc_failure() here, because we cannot block.
- // The allocation failure would be handled by the WB slowpath with handle_alloc_failure_evac().
+ // The allocation failure would be handled by the LRB slowpath with handle_alloc_failure_evac().
}
if (in_new_region) {
@@ -1105,7 +1101,6 @@
ShenandoahParallelWorkerSession worker_session(worker_id);
ShenandoahEvacOOMScope oom_evac_scope;
ShenandoahEvacuateUpdateRootsClosure cl;
-
MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
_rp->process_evacuate_roots(&cl, &blobsCl, worker_id);
}
@@ -2062,14 +2057,12 @@
}
oop ShenandoahHeap::pin_object(JavaThread* thr, oop o) {
- o = ShenandoahBarrierSet::barrier_set()->write_barrier(o);
ShenandoahHeapLocker locker(lock());
heap_region_containing(o)->make_pinned();
return o;
}
void ShenandoahHeap::unpin_object(JavaThread* thr, oop o) {
- o = ShenandoahBarrierSet::barrier_set()->read_barrier(o);
ShenandoahHeapLocker locker(lock());
heap_region_containing(o)->make_unpinned();
}
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp Tue Apr 02 23:00:22 2019 +0200
@@ -270,16 +270,16 @@
//
public:
enum GCStateBitPos {
- // Heap has forwarded objects: need RB, ACMP, CAS barriers.
+ // Heap has forwarded objects: needs LRB barriers.
HAS_FORWARDED_BITPOS = 0,
// Heap is under marking: needs SATB barriers.
MARKING_BITPOS = 1,
- // Heap is under evacuation: needs WB barriers. (Set together with UNSTABLE)
+ // Heap is under evacuation: needs LRB barriers. (Set together with HAS_FORWARDED)
EVACUATION_BITPOS = 2,
- // Heap is under updating: needs SVRB/SVWB barriers.
+ // Heap is under updating: needs no additional barriers.
UPDATEREFS_BITPOS = 3,
// Heap is under traversal collection
--- a/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp Tue Apr 02 23:00:22 2019 +0200
@@ -129,7 +129,7 @@
// Once marking is done, which may have fixed up forwarded objects, we can drop it.
// Coming out of Full GC, we would not have any forwarded objects.
- // This also prevents read barrier from kicking in while adjusting pointers in phase3.
+ // This also prevents resolves with fwdptr from kicking in while adjusting pointers in phase3.
heap->set_has_forwarded_objects(false);
heap->set_full_gc_move_in_progress(true);
--- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp Tue Apr 02 23:00:22 2019 +0200
@@ -34,7 +34,7 @@
enum UpdateRefsMode {
NONE, // No reference updating
- RESOLVE, // Only a read-barrier (no reference updating)
+ RESOLVE, // Only a resolve (no reference updating)
SIMPLE, // Reference updating using simple store
CONCURRENT // Reference updating using CAS
};
--- a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp Tue Apr 02 23:00:22 2019 +0200
@@ -242,13 +242,21 @@
_evacuation_tasks(new SubTasksDone(SHENANDOAH_EVAC_NumElements)),
_srs(n_workers),
_phase(phase),
- _coderoots_cset_iterator(ShenandoahCodeRoots::cset_iterator())
+ _coderoots_cset_iterator(ShenandoahCodeRoots::cset_iterator()),
+ _par_state_string(StringTable::weak_storage())
+
{
heap->phase_timings()->record_workers_start(_phase);
+ if (ShenandoahStringDedup::is_enabled()) {
+ StringDedup::gc_prologue(false);
+ }
}
ShenandoahRootEvacuator::~ShenandoahRootEvacuator() {
delete _evacuation_tasks;
+ if (ShenandoahStringDedup::is_enabled()) {
+ StringDedup::gc_epilogue();
+ }
ShenandoahHeap::heap()->phase_timings()->record_workers_end(_phase);
}
@@ -270,11 +278,38 @@
_coderoots_cset_iterator.possibly_parallel_blobs_do(blobs);
}
- if (_evacuation_tasks->try_claim_task(SHENANDOAH_EVAC_jvmti_oops_do)) {
+ if (ShenandoahStringDedup::is_enabled()) {
ShenandoahForwardedIsAliveClosure is_alive;
+ ShenandoahStringDedup::parallel_oops_do(&is_alive, oops, worker_id);
+ }
+
+ if (_evacuation_tasks->try_claim_task(SHENANDOAH_EVAC_Universe_oops_do)) {
+ ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::UniverseRoots, worker_id);
+ Universe::oops_do(oops);
+ }
+
+ if (_evacuation_tasks->try_claim_task(SHENANDOAH_EVAC_Management_oops_do)) {
+ ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::ManagementRoots, worker_id);
+ Management::oops_do(oops);
+ }
+
+ if (_evacuation_tasks->try_claim_task(SHENANDOAH_EVAC_jvmti_oops_do)) {
ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::JVMTIRoots, worker_id);
+ JvmtiExport::oops_do(oops);
+ ShenandoahForwardedIsAliveClosure is_alive;
JvmtiExport::weak_oops_do(&is_alive, oops);
}
+
+ if (_evacuation_tasks->try_claim_task(SHENANDOAH_EVAC_SystemDictionary_oops_do)) {
+ ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::SystemDictionaryRoots, worker_id);
+ SystemDictionary::oops_do(oops);
+ }
+
+ if (_evacuation_tasks->try_claim_task(SHENANDOAH_EVAC_ObjectSynchronizer_oops_do)) {
+ ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::ObjectSynchronizerRoots, worker_id);
+ ObjectSynchronizer::oops_do(oops);
+ }
+
}
uint ShenandoahRootEvacuator::n_workers() const {
--- a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp Tue Apr 02 23:00:22 2019 +0200
@@ -58,7 +58,7 @@
StrongRootsScope _srs;
OopStorage::ParState<false, false> _par_state_string;
ShenandoahPhaseTimings::Phase _phase;
- ParallelCLDRootIterator _cld_iterator;
+ ParallelCLDRootIterator _cld_iterator;
ShenandoahAllCodeRootsIterator _coderoots_all_iterator;
CodeBlobClosure* _threads_nmethods_cl;
WeakProcessorPhaseTimes _weak_processor_timings;
@@ -120,11 +120,16 @@
StrongRootsScope _srs;
ShenandoahPhaseTimings::Phase _phase;
ShenandoahCsetCodeRootsIterator _coderoots_cset_iterator;
+ OopStorage::ParState<false, false> _par_state_string;
enum Shenandoah_evacuate_roots_tasks {
- SHENANDOAH_EVAC_jvmti_oops_do,
- // Leave this one last.
- SHENANDOAH_EVAC_NumElements
+ SHENANDOAH_EVAC_Universe_oops_do,
+ SHENANDOAH_EVAC_ObjectSynchronizer_oops_do,
+ SHENANDOAH_EVAC_Management_oops_do,
+ SHENANDOAH_EVAC_SystemDictionary_oops_do,
+ SHENANDOAH_EVAC_jvmti_oops_do,
+ // Leave this one last.
+ SHENANDOAH_EVAC_NumElements
};
public:
ShenandoahRootEvacuator(ShenandoahHeap* heap, uint n_workers,
--- a/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp Tue Apr 02 23:00:22 2019 +0200
@@ -55,8 +55,8 @@
ShenandoahThreadLocalData::satb_mark_queue(thread).enqueue_known_active(orig);
JRT_END
-JRT_LEAF(oopDesc*, ShenandoahRuntime::write_barrier_JRT(oopDesc* src))
- oop result = ShenandoahBarrierSet::barrier_set()->write_barrier_mutator(src);
+JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_JRT(oopDesc* src))
+ oop result = ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator(src);
return (oopDesc*) result;
JRT_END
--- a/src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp Tue Apr 02 23:00:22 2019 +0200
@@ -37,7 +37,7 @@
static void write_ref_array_post_entry(HeapWord* dst, size_t length);
static void write_ref_field_pre_entry(oopDesc* orig, JavaThread* thread);
- static oopDesc* write_barrier_JRT(oopDesc* src);
+ static oopDesc* load_reference_barrier_JRT(oopDesc* src);
static void shenandoah_clone_barrier(oopDesc* obj);
};
--- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp Tue Apr 02 23:00:22 2019 +0200
@@ -244,7 +244,7 @@
"Time is in microseconds.") \
\
experimental(uintx, ShenandoahEvacAssist, 10, \
- "How many objects to evacuate on WB assist path. " \
+ "How many objects to evacuate on LRB assist path. " \
"Use zero to disable.") \
\
experimental(bool, ShenandoahPacing, true, \
@@ -352,27 +352,18 @@
diagnostic(bool, ShenandoahKeepAliveBarrier, true, \
"Turn on/off keep alive barriers in Shenandoah") \
\
- diagnostic(bool, ShenandoahWriteBarrier, true, \
- "Turn on/off write barriers in Shenandoah") \
- \
- diagnostic(bool, ShenandoahReadBarrier, true, \
- "Turn on/off read barriers in Shenandoah") \
- \
diagnostic(bool, ShenandoahStoreValEnqueueBarrier, false, \
"Turn on/off enqueuing of oops for storeval barriers") \
\
- diagnostic(bool, ShenandoahStoreValReadBarrier, true, \
- "Turn on/off store val read barriers in Shenandoah") \
- \
diagnostic(bool, ShenandoahCASBarrier, true, \
"Turn on/off CAS barriers in Shenandoah") \
\
- diagnostic(bool, ShenandoahAcmpBarrier, true, \
- "Turn on/off acmp barriers in Shenandoah") \
- \
diagnostic(bool, ShenandoahCloneBarrier, true, \
"Turn on/off clone barriers in Shenandoah") \
\
+ diagnostic(bool, ShenandoahLoadRefBarrier, true, \
+ "Turn on/off load-reference barriers in Shenandoah") \
+ \
diagnostic(bool, ShenandoahStoreCheck, false, \
"Emit additional code that checks objects are written to only" \
" in to-space") \
@@ -401,20 +392,13 @@
"Turn it off for maximum compatibility with reflection or JNI " \
"code that manipulates final fields.") \
\
- diagnostic(bool, ShenandoahDecreaseRegisterPressure, false, \
- "Try to reuse after-barrier values to reduce register pressure") \
- \
experimental(bool, ShenandoahCommonGCStateLoads, false, \
"Enable commonming for GC state loads in generated code.") \
\
develop(bool, ShenandoahVerifyOptoBarriers, false, \
"Verify no missing barriers in C2") \
\
- experimental(bool, ShenandoahDontIncreaseWBFreq, true, \
- "Common 2 WriteBarriers or WriteBarrier and a ReadBarrier only " \
- "if the resulting WriteBarrier isn't executed more frequently") \
- \
experimental(bool, ShenandoahLoopOptsAfterExpansion, true, \
- "Attempt more loop opts after write barrier expansion") \
+ "Attempt more loop opts after barrier expansion") \
#endif // SHARE_GC_SHENANDOAH_SHENANDOAH_GLOBALS_HPP
--- a/src/hotspot/share/opto/classes.hpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/opto/classes.hpp Tue Apr 02 23:00:22 2019 +0200
@@ -281,9 +281,7 @@
shmacro(ShenandoahWeakCompareAndSwapN)
shmacro(ShenandoahWeakCompareAndSwapP)
shmacro(ShenandoahEnqueueBarrier)
-shmacro(ShenandoahReadBarrier)
-shmacro(ShenandoahWriteBarrier)
-shmacro(ShenandoahWBMemProj)
+shmacro(ShenandoahLoadReferenceBarrier)
macro(SCMemProj)
macro(SqrtD)
macro(SqrtF)
--- a/src/hotspot/share/opto/compile.cpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/opto/compile.cpp Tue Apr 02 23:00:22 2019 +0200
@@ -3070,7 +3070,7 @@
Node *m = wq.at(next);
for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
Node* use = m->fast_out(i);
- if (use->is_Mem() || use->is_EncodeNarrowPtr() || use->is_ShenandoahBarrier()) {
+ if (use->is_Mem() || use->is_EncodeNarrowPtr()) {
use->ensure_control_or_add_prec(n->in(0));
} else {
switch(use->Opcode()) {
--- a/src/hotspot/share/opto/lcm.cpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/opto/lcm.cpp Tue Apr 02 23:00:22 2019 +0200
@@ -178,7 +178,6 @@
case Op_LoadRange:
case Op_LoadD_unaligned:
case Op_LoadL_unaligned:
- case Op_ShenandoahReadBarrier:
assert(mach->in(2) == val, "should be address");
break;
case Op_StoreB:
--- a/src/hotspot/share/opto/library_call.cpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/opto/library_call.cpp Tue Apr 02 23:00:22 2019 +0200
@@ -4485,7 +4485,7 @@
for (MergeMemStream mms(merged_memory(), mem->as_MergeMem()); mms.next_non_empty2(); ) {
Node* n = mms.memory();
if (n != mms.memory2() && !(n->is_Proj() && n->in(0) == alloc->initialization())) {
- assert(n->is_Store() || n->Opcode() == Op_ShenandoahWBMemProj, "what else?");
+ assert(n->is_Store(), "what else?");
no_interfering_store = false;
break;
}
@@ -4494,7 +4494,7 @@
for (MergeMemStream mms(merged_memory()); mms.next_non_empty(); ) {
Node* n = mms.memory();
if (n != mem && !(n->is_Proj() && n->in(0) == alloc->initialization())) {
- assert(n->is_Store() || n->Opcode() == Op_ShenandoahWBMemProj, "what else?");
+ assert(n->is_Store(), "what else?");
no_interfering_store = false;
break;
}
--- a/src/hotspot/share/opto/loopPredicate.cpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/opto/loopPredicate.cpp Tue Apr 02 23:00:22 2019 +0200
@@ -536,9 +536,6 @@
if (_lpt->is_invariant(n)) { // known invariant
_invariant.set(n->_idx);
} else if (!n->is_CFG()) {
- if (n->Opcode() == Op_ShenandoahWriteBarrier) {
- return;
- }
Node *n_ctrl = _phase->ctrl_or_self(n);
Node *u_ctrl = _phase->ctrl_or_self(use); // self if use is a CFG
if (_phase->is_dominator(n_ctrl, u_ctrl)) {
--- a/src/hotspot/share/opto/loopnode.cpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/opto/loopnode.cpp Tue Apr 02 23:00:22 2019 +0200
@@ -3971,7 +3971,7 @@
}
while(worklist.size() != 0 && LCA != early) {
Node* s = worklist.pop();
- if (s->is_Load() || s->is_ShenandoahBarrier() || s->Opcode() == Op_SafePoint ||
+ if (s->is_Load() || s->Opcode() == Op_SafePoint ||
(s->is_CallStaticJava() && s->as_CallStaticJava()->uncommon_trap_request() != 0)) {
continue;
} else if (s->is_MergeMem()) {
--- a/src/hotspot/share/opto/loopnode.hpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/opto/loopnode.hpp Tue Apr 02 23:00:22 2019 +0200
@@ -38,8 +38,6 @@
class LoopNode;
class Node;
class OuterStripMinedLoopEndNode;
-class ShenandoahBarrierNode;
-class ShenandoahWriteBarrierNode;
class PathFrequency;
class PhaseIdealLoop;
class CountedLoopReserveKit;
@@ -638,8 +636,7 @@
friend class IdealLoopTree;
friend class SuperWord;
friend class CountedLoopReserveKit;
- friend class ShenandoahBarrierNode;
- friend class ShenandoahWriteBarrierNode;
+ friend class ShenandoahBarrierC2Support;
// Pre-computed def-use info
PhaseIterGVN &_igvn;
--- a/src/hotspot/share/opto/loopopts.cpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/opto/loopopts.cpp Tue Apr 02 23:00:22 2019 +0200
@@ -1082,11 +1082,6 @@
Node* m = n->fast_out(j);
if (m->is_FastLock())
return false;
-#if INCLUDE_SHENANDOAHGC
- if (m->is_ShenandoahBarrier() && m->has_out_with(Op_FastLock)) {
- return false;
- }
-#endif
#ifdef _LP64
if (m->Opcode() == Op_ConvI2L)
return false;
@@ -3210,7 +3205,7 @@
// if not pinned and not a load (which maybe anti-dependent on a store)
// and not a CMove (Matcher expects only bool->cmove).
- if (n->in(0) == NULL && !n->is_Load() && !n->is_CMove() && n->Opcode() != Op_ShenandoahWBMemProj) {
+ if (n->in(0) == NULL && !n->is_Load() && !n->is_CMove()) {
cloned_for_outside_use += clone_for_use_outside_loop( loop, n, worklist );
sink_list.push(n);
peel >>= n->_idx; // delete n from peel set.
--- a/src/hotspot/share/opto/node.hpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/share/opto/node.hpp Tue Apr 02 23:00:22 2019 +0200
@@ -142,7 +142,6 @@
class RootNode;
class SafePointNode;
class SafePointScalarObjectNode;
-class ShenandoahBarrierNode;
class StartNode;
class State;
class StoreNode;
@@ -676,7 +675,6 @@
DEFINE_CLASS_ID(EncodeNarrowPtr, Type, 6)
DEFINE_CLASS_ID(EncodeP, EncodeNarrowPtr, 0)
DEFINE_CLASS_ID(EncodePKlass, EncodeNarrowPtr, 1)
- DEFINE_CLASS_ID(ShenandoahBarrier, Type, 7)
DEFINE_CLASS_ID(Proj, Node, 3)
DEFINE_CLASS_ID(CatchProj, Proj, 0)
@@ -875,7 +873,6 @@
DEFINE_CLASS_QUERY(Root)
DEFINE_CLASS_QUERY(SafePoint)
DEFINE_CLASS_QUERY(SafePointScalarObject)
- DEFINE_CLASS_QUERY(ShenandoahBarrier)
DEFINE_CLASS_QUERY(Start)
DEFINE_CLASS_QUERY(Store)
DEFINE_CLASS_QUERY(Sub)
--- a/test/hotspot/jtreg/gc/shenandoah/options/TestSelectiveBarrierFlags.java Thu Apr 04 07:43:44 2019 -0700
+++ b/test/hotspot/jtreg/gc/shenandoah/options/TestSelectiveBarrierFlags.java Tue Apr 02 23:00:22 2019 +0200
@@ -43,13 +43,9 @@
public static void main(String[] args) throws Exception {
String[][] opts = {
new String[] { "ShenandoahKeepAliveBarrier" },
- new String[] { "ShenandoahWriteBarrier" },
- new String[] { "ShenandoahReadBarrier" },
- // StoreValRead+SATB are actually compatible, but we need to protect against
- // StorveValEnqueue+SATB. TODO: Make it better.
- new String[] { "ShenandoahSATBBarrier", "ShenandoahStoreValReadBarrier", "ShenandoahStoreValEnqueueBarrier" },
+ new String[] { "ShenandoahLoadRefBarrier" },
+ new String[] { "ShenandoahSATBBarrier", "ShenandoahStoreValEnqueueBarrier" },
new String[] { "ShenandoahCASBarrier" },
- new String[] { "ShenandoahAcmpBarrier" },
new String[] { "ShenandoahCloneBarrier" },
};
--- a/test/hotspot/jtreg/gc/shenandoah/options/TestWrongBarrierDisable.java Thu Apr 04 07:43:44 2019 -0700
+++ b/test/hotspot/jtreg/gc/shenandoah/options/TestWrongBarrierDisable.java Tue Apr 02 23:00:22 2019 +0200
@@ -38,21 +38,16 @@
public static void main(String[] args) throws Exception {
String[] concurrent = {
- "ShenandoahReadBarrier",
- "ShenandoahWriteBarrier",
+ "ShenandoahLoadRefBarrier",
"ShenandoahCASBarrier",
- "ShenandoahAcmpBarrier",
"ShenandoahCloneBarrier",
"ShenandoahSATBBarrier",
"ShenandoahKeepAliveBarrier",
- "ShenandoahStoreValReadBarrier",
};
String[] traversal = {
- "ShenandoahReadBarrier",
- "ShenandoahWriteBarrier",
+ "ShenandoahLoadRefBarrier",
"ShenandoahCASBarrier",
- "ShenandoahAcmpBarrier",
"ShenandoahCloneBarrier",
};