--- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp Thu Apr 04 07:43:44 2019 -0700
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp Tue Apr 02 23:00:22 2019 +0200
@@ -40,7 +40,7 @@
#define __ masm->
-address ShenandoahBarrierSetAssembler::_shenandoah_wb = NULL;
+address ShenandoahBarrierSetAssembler::_shenandoah_lrb = NULL;
void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
Register addr, Register count, RegSet saved_regs) {
@@ -198,60 +198,31 @@
__ bind(done);
}
-void ShenandoahBarrierSetAssembler::read_barrier(MacroAssembler* masm, Register dst) {
- if (ShenandoahReadBarrier) {
- read_barrier_impl(masm, dst);
- }
-}
-
-void ShenandoahBarrierSetAssembler::read_barrier_impl(MacroAssembler* masm, Register dst) {
- assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
+void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst) {
+ assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled");
Label is_null;
__ cbz(dst, is_null);
- read_barrier_not_null_impl(masm, dst);
+ resolve_forward_pointer_not_null(masm, dst);
__ bind(is_null);
}
-void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) {
- if (ShenandoahReadBarrier) {
- read_barrier_not_null_impl(masm, dst);
- }
-}
-
-
-void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) {
- assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
+// IMPORTANT: This must preserve all registers, even rscratch1 and rscratch2.
+void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst) {
+ assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled");
__ ldr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset()));
}
-void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) {
- if (ShenandoahWriteBarrier) {
- write_barrier_impl(masm, dst);
- }
-}
-
-void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) {
- assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "Should be enabled");
- assert(dst != rscratch1, "need rscratch1");
+void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst, Register tmp) {
+ assert(ShenandoahLoadRefBarrier, "Should be enabled");
assert(dst != rscratch2, "need rscratch2");
Label done;
-
+ __ enter();
Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
- __ ldrb(rscratch1, gc_state);
+ __ ldrb(rscratch2, gc_state);
// Check for heap stability
- __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
- __ tst(rscratch1, rscratch2);
- __ br(Assembler::EQ, done);
-
- // Heap is unstable, need to perform the read-barrier even if WB is inactive
- __ ldr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset()));
-
- // Check for evacuation-in-progress and jump to WB slow-path if needed
- __ mov(rscratch2, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
- __ tst(rscratch1, rscratch2);
- __ br(Assembler::EQ, done);
+ __ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
RegSet to_save = RegSet::of(r0);
if (dst != r0) {
@@ -259,7 +230,7 @@
__ mov(r0, dst);
}
- __ far_call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_wb())));
+ __ far_call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_lrb())));
if (dst != r0) {
__ mov(dst, r0);
@@ -267,14 +238,11 @@
}
__ bind(done);
+ __ leave();
}
void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
if (ShenandoahStoreValEnqueueBarrier) {
- Label is_null;
- __ cbz(dst, is_null);
- write_barrier_impl(masm, dst);
- __ bind(is_null);
// Save possibly live regs.
RegSet live_regs = RegSet::range(r0, r4) - dst;
__ push(live_regs, sp);
@@ -286,44 +254,45 @@
__ ldrd(v0, __ post(sp, 2 * wordSize));
__ pop(live_regs, sp);
}
- if (ShenandoahStoreValReadBarrier) {
- read_barrier_impl(masm, dst);
+}
+
+void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Register tmp) {
+ if (ShenandoahLoadRefBarrier) {
+ Label is_null;
+ __ cbz(dst, is_null);
+ load_reference_barrier_not_null(masm, dst, tmp);
+ __ bind(is_null);
}
}
void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register dst, Address src, Register tmp1, Register tmp_thread) {
bool on_oop = type == T_OBJECT || type == T_ARRAY;
- bool in_heap = (decorators & IN_HEAP) != 0;
bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
bool on_reference = on_weak || on_phantom;
- if (in_heap) {
- read_barrier_not_null(masm, src.base());
- }
+ BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
+ if (on_oop) {
+ load_reference_barrier(masm, dst, tmp1);
- BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
- if (ShenandoahKeepAliveBarrier && on_oop && on_reference) {
- __ enter();
- satb_write_barrier_pre(masm /* masm */,
- noreg /* obj */,
- dst /* pre_val */,
- rthread /* thread */,
- tmp1 /* tmp */,
- true /* tosca_live */,
- true /* expand_call */);
- __ leave();
+ if (ShenandoahKeepAliveBarrier && on_reference) {
+ __ enter();
+ satb_write_barrier_pre(masm /* masm */,
+ noreg /* obj */,
+ dst /* pre_val */,
+ rthread /* thread */,
+ tmp1 /* tmp */,
+ true /* tosca_live */,
+ true /* expand_call */);
+ __ leave();
+ }
}
}
void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Address dst, Register val, Register tmp1, Register tmp2) {
bool on_oop = type == T_OBJECT || type == T_ARRAY;
- bool in_heap = (decorators & IN_HEAP) != 0;
- if (in_heap) {
- write_barrier(masm, dst.base());
- }
if (!on_oop) {
BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
return;
@@ -361,21 +330,6 @@
}
-void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register op1, Register op2) {
- __ cmp(op1, op2);
- if (ShenandoahAcmpBarrier) {
- Label done;
- __ br(Assembler::EQ, done);
- // The object may have been evacuated, but we won't see it without a
- // membar here.
- __ membar(Assembler::LoadStore| Assembler::LoadLoad);
- read_barrier(masm, op1);
- read_barrier(masm, op2);
- __ cmp(op1, op2);
- __ bind(done);
- }
-}
-
void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm, Register obj,
Register var_size_in_bytes,
int con_size_in_bytes,
@@ -410,27 +364,6 @@
}
}
-void ShenandoahBarrierSetAssembler::resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj) {
- bool oop_not_null = (decorators & IS_NOT_NULL) != 0;
- bool is_write = (decorators & ACCESS_WRITE) != 0;
- if (is_write) {
- if (oop_not_null) {
- write_barrier(masm, obj);
- } else {
- Label done;
- __ cbz(obj, done);
- write_barrier(masm, obj);
- __ bind(done);
- }
- } else {
- if (oop_not_null) {
- read_barrier_not_null(masm, obj);
- } else {
- read_barrier(masm, obj);
- }
- }
-}
-
void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val,
bool acquire, bool release, bool weak, bool is_cae,
Register result) {
@@ -469,8 +402,8 @@
__ decode_heap_oop(tmp1, tmp1);
__ decode_heap_oop(tmp2, tmp2);
}
- read_barrier_impl(masm, tmp1);
- read_barrier_impl(masm, tmp2);
+ resolve_forward_pointer(masm, tmp1);
+ resolve_forward_pointer(masm, tmp2);
__ cmp(tmp1, tmp2);
// Retry with expected now being the value we just loaded from addr.
__ br(Assembler::EQ, retry);
@@ -515,7 +448,7 @@
__ b(*stub->continuation());
}
-void ShenandoahBarrierSetAssembler::gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub) {
+void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
Register obj = stub->obj()->as_register();
Register res = stub->result()->as_register();
@@ -532,7 +465,7 @@
__ cbz(res, done);
}
- write_barrier(ce->masm(), res);
+ load_reference_barrier_not_null(ce->masm(), res, rscratch1);
__ bind(done);
__ b(*stub->continuation());
@@ -592,14 +525,14 @@
#endif // COMPILER1
-address ShenandoahBarrierSetAssembler::shenandoah_wb() {
- assert(_shenandoah_wb != NULL, "need write barrier stub");
- return _shenandoah_wb;
+address ShenandoahBarrierSetAssembler::shenandoah_lrb() {
+ assert(_shenandoah_lrb != NULL, "need load reference barrier stub");
+ return _shenandoah_lrb;
}
#define __ cgen->assembler()->
-// Shenandoah write barrier.
+// Shenandoah load reference barrier.
//
// Input:
// r0: OOP to evacuate. Not null.
@@ -608,13 +541,13 @@
// r0: Pointer to evacuated OOP.
//
// Trash rscratch1, rscratch2. Preserve everything else.
-address ShenandoahBarrierSetAssembler::generate_shenandoah_wb(StubCodeGenerator* cgen) {
+address ShenandoahBarrierSetAssembler::generate_shenandoah_lrb(StubCodeGenerator* cgen) {
__ align(6);
- StubCodeMark mark(cgen, "StubRoutines", "shenandoah_wb");
+ StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb");
address start = __ pc();
- Label work;
+ Label work, done;
__ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
__ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
__ ldrb(rscratch2, Address(rscratch2, rscratch1));
@@ -622,19 +555,23 @@
__ ret(lr);
__ bind(work);
- Register obj = r0;
+ __ mov(rscratch2, r0);
+ resolve_forward_pointer_not_null(cgen->assembler(), r0);
+ __ cmp(rscratch2, r0);
+ __ br(Assembler::NE, done);
__ enter(); // required for proper stackwalking of RuntimeStub frame
__ push_call_clobbered_registers();
- __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT));
+ __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT));
__ blrt(lr, 1, 0, MacroAssembler::ret_type_integral);
- __ mov(rscratch1, obj);
+ __ mov(rscratch1, r0);
__ pop_call_clobbered_registers();
- __ mov(obj, rscratch1);
+ __ mov(r0, rscratch1);
__ leave(); // required for proper stackwalking of RuntimeStub frame
+ __ bind(done);
__ ret(lr);
return start;
@@ -643,12 +580,12 @@
#undef __
void ShenandoahBarrierSetAssembler::barrier_stubs_init() {
- if (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) {
+ if (ShenandoahLoadRefBarrier) {
int stub_code_size = 2048;
ResourceMark rm;
BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size);
CodeBuffer buf(bb);
StubCodeGenerator cgen(&buf);
- _shenandoah_wb = generate_shenandoah_wb(&cgen);
+ _shenandoah_lrb = generate_shenandoah_lrb(&cgen);
}
}