--- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp Thu Oct 17 20:27:44 2019 +0100
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp Thu Oct 17 20:53:35 2019 +0100
@@ -24,7 +24,7 @@
#include "precompiled.hpp"
#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
#include "gc/shenandoah/shenandoahForwarding.hpp"
-#include "gc/shenandoah/shenandoahHeap.hpp"
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.hpp"
#include "gc/shenandoah/shenandoahHeuristics.hpp"
#include "gc/shenandoah/shenandoahRuntime.hpp"
@@ -47,7 +47,7 @@
Register src, Register dst, Register count, RegSet saved_regs) {
if (is_oop) {
bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
- if (ShenandoahSATBBarrier && !dest_uninitialized && !ShenandoahHeap::heap()->heuristics()->can_do_traversal_gc()) {
+ if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
Label done;
@@ -57,27 +57,27 @@
// Is marking active?
Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
__ ldrb(rscratch1, gc_state);
- __ tbz(rscratch1, ShenandoahHeap::MARKING_BITPOS, done);
+ if (dest_uninitialized) {
+ __ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
+ } else {
+ __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
+ __ tst(rscratch1, rscratch2);
+ __ br(Assembler::EQ, done);
+ }
__ push(saved_regs, sp);
- if (count == c_rarg0) {
- if (dst == c_rarg1) {
- // exactly backwards!!
- __ mov(rscratch1, c_rarg0);
- __ mov(c_rarg0, c_rarg1);
- __ mov(c_rarg1, rscratch1);
+ if (UseCompressedOops) {
+ if (dest_uninitialized) {
+ __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_narrow_oop_entry), src, dst, count);
} else {
- __ mov(c_rarg1, count);
- __ mov(c_rarg0, dst);
+ __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), src, dst, count);
}
} else {
- __ mov(c_rarg0, dst);
- __ mov(c_rarg1, count);
- }
- if (UseCompressedOops) {
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), 2);
- } else {
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), 2);
+ if (dest_uninitialized) {
+ __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_oop_entry), src, dst, count);
+ } else {
+ __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), src, dst, count);
+ }
}
__ pop(saved_regs, sp);
__ bind(done);
@@ -85,31 +85,6 @@
}
}
-void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
- Register start, Register count, Register scratch, RegSet saved_regs) {
- if (is_oop) {
- Label done;
-
- // Avoid calling runtime if count == 0
- __ cbz(count, done);
-
- // Is updating references?
- Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
- __ ldrb(rscratch1, gc_state);
- __ tbz(rscratch1, ShenandoahHeap::UPDATEREFS_BITPOS, done);
-
- __ push(saved_regs, sp);
- assert_different_registers(start, count, scratch);
- assert_different_registers(c_rarg0, count);
- __ mov(c_rarg0, start);
- __ mov(c_rarg1, count);
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry), 2);
- __ pop(saved_regs, sp);
-
- __ bind(done);
- }
-}
-
void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
Register obj,
Register pre_val,
@@ -236,15 +211,20 @@
if (borrow_reg) {
// No free registers available. Make one useful.
tmp = rscratch1;
+ if (tmp == dst) {
+ tmp = rscratch2;
+ }
__ push(RegSet::of(tmp), sp);
}
+ assert_different_registers(tmp, dst);
+
Label done;
__ ldr(tmp, Address(dst, oopDesc::mark_offset_in_bytes()));
__ eon(tmp, tmp, zr);
- __ ands(zr, tmp, markOopDesc::lock_mask_in_place);
+ __ ands(zr, tmp, markWord::lock_mask_in_place);
__ br(Assembler::NE, done);
- __ orr(tmp, tmp, markOopDesc::marked_value);
+ __ orr(tmp, tmp, markWord::marked_value);
__ eon(dst, tmp, zr);
__ bind(done);
@@ -282,6 +262,40 @@
__ leave();
}
+void ShenandoahBarrierSetAssembler::load_reference_barrier_native(MacroAssembler* masm, Register dst, Register tmp) {
+ if (!ShenandoahLoadRefBarrier) {
+ return;
+ }
+
+ assert(dst != rscratch2, "need rscratch2");
+
+ Label is_null;
+ Label done;
+
+ __ cbz(dst, is_null);
+
+ __ enter();
+
+ Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+ __ ldrb(rscratch2, gc_state);
+
+ // Check for heap in evacuation phase
+ __ tbz(rscratch2, ShenandoahHeap::EVACUATION_BITPOS, done);
+
+ __ mov(rscratch2, dst);
+ __ push_call_clobbered_registers();
+ __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native));
+ __ mov(r0, rscratch2);
+ __ blr(lr);
+ __ mov(rscratch2, r0);
+ __ pop_call_clobbered_registers();
+ __ mov(dst, rscratch2);
+
+ __ bind(done);
+ __ leave();
+ __ bind(is_null);
+}
+
void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
if (ShenandoahStoreValEnqueueBarrier) {
// Save possibly live regs.
@@ -308,16 +322,26 @@
void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register dst, Address src, Register tmp1, Register tmp_thread) {
- bool on_oop = type == T_OBJECT || type == T_ARRAY;
+ bool on_oop = is_reference_type(type);
+ bool not_in_heap = (decorators & IN_NATIVE) != 0;
bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
bool on_reference = on_weak || on_phantom;
+ bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0;
BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
if (on_oop) {
- load_reference_barrier(masm, dst, tmp1);
-
- if (ShenandoahKeepAliveBarrier && on_reference) {
+ if (not_in_heap) {
+ if (ShenandoahHeap::heap()->is_traversal_mode()) {
+ load_reference_barrier(masm, dst, tmp1);
+ keep_alive = true;
+ } else {
+ load_reference_barrier_native(masm, dst, tmp1);
+ }
+ } else {
+ load_reference_barrier(masm, dst, tmp1);
+ }
+ if (ShenandoahKeepAliveBarrier && on_reference && keep_alive) {
__ enter();
satb_write_barrier_pre(masm /* masm */,
noreg /* obj */,
@@ -333,7 +357,7 @@
void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Address dst, Register val, Register tmp1, Register tmp2) {
- bool on_oop = type == T_OBJECT || type == T_ARRAY;
+ bool on_oop = is_reference_type(type);
if (!on_oop) {
BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
return;
@@ -371,6 +395,26 @@
}
+void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
+ Register obj, Register tmp, Label& slowpath) {
+ Label done;
+ // Resolve jobject
+ BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
+
+ // Check for null.
+ __ cbz(obj, done);
+
+ assert(obj != rscratch2, "need rscratch2");
+ Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+ __ ldrb(rscratch2, gc_state);
+
+ // Check for heap in evacuation phase
+ __ tbnz(rscratch2, ShenandoahHeap::EVACUATION_BITPOS, slowpath);
+
+ __ bind(done);
+}
+
+
void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val,
bool acquire, bool release, bool weak, bool is_cae,
Register result) {
@@ -457,25 +501,47 @@
}
void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
+ ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
+ __ bind(*stub->entry());
Register obj = stub->obj()->as_register();
Register res = stub->result()->as_register();
+ Register addr = stub->addr()->as_register_lo();
+ Register tmp1 = stub->tmp1()->as_register();
+ Register tmp2 = stub->tmp2()->as_register();
- Label done;
-
- __ bind(*stub->entry());
+ assert(res == r0, "result must arrive in r0");
if (res != obj) {
__ mov(res, obj);
}
+
// Check for null.
- if (stub->needs_null_check()) {
- __ cbz(res, done);
- }
+ __ cbz(res, *stub->continuation());
+
+ // Check for object in cset.
+ __ mov(tmp2, ShenandoahHeap::in_cset_fast_test_addr());
+ __ lsr(tmp1, res, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+ __ ldrb(tmp2, Address(tmp2, tmp1));
+ __ cbz(tmp2, *stub->continuation());
- load_reference_barrier_not_null(ce->masm(), res, rscratch1);
+ // Check if object is already forwarded.
+ Label slow_path;
+ __ ldr(tmp1, Address(res, oopDesc::mark_offset_in_bytes()));
+ __ eon(tmp1, tmp1, zr);
+ __ ands(zr, tmp1, markWord::lock_mask_in_place);
+ __ br(Assembler::NE, slow_path);
- __ bind(done);
+ // Decode forwarded object.
+ __ orr(tmp1, tmp1, markWord::marked_value);
+ __ eon(res, tmp1, zr);
+ __ b(*stub->continuation());
+
+ __ bind(slow_path);
+ ce->store_parameter(res, 0);
+ ce->store_parameter(addr, 1);
+ __ far_call(RuntimeAddress(bs->load_reference_barrier_rt_code_blob()->code_begin()));
+
__ b(*stub->continuation());
}
@@ -529,6 +595,26 @@
__ epilogue();
}
+void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm) {
+ __ prologue("shenandoah_load_reference_barrier", false);
+ // arg0 : object to be resolved
+
+ __ push_call_clobbered_registers();
+ __ load_parameter(0, r0);
+ __ load_parameter(1, r1);
+ if (UseCompressedOops) {
+ __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup_narrow));
+ } else {
+ __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup));
+ }
+ __ blr(lr);
+ __ mov(rscratch1, r0);
+ __ pop_call_clobbered_registers();
+ __ mov(r0, rscratch1);
+
+ __ epilogue();
+}
+
#undef __
#endif // COMPILER1
@@ -563,17 +649,24 @@
__ ret(lr);
__ bind(work);
- __ mov(rscratch2, r0);
- resolve_forward_pointer_not_null(cgen->assembler(), r0, rscratch1);
- __ cmp(rscratch2, r0);
- __ br(Assembler::NE, done);
+ Label slow_path;
+ __ ldr(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes()));
+ __ eon(rscratch1, rscratch1, zr);
+ __ ands(zr, rscratch1, markWord::lock_mask_in_place);
+ __ br(Assembler::NE, slow_path);
+ // Decode forwarded object.
+ __ orr(rscratch1, rscratch1, markWord::marked_value);
+ __ eon(r0, rscratch1, zr);
+ __ ret(lr);
+
+ __ bind(slow_path);
__ enter(); // required for proper stackwalking of RuntimeStub frame
__ push_call_clobbered_registers();
- __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT));
- __ blrt(lr, 1, 0, MacroAssembler::ret_type_integral);
+ __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier));
+ __ blr(lr);
__ mov(rscratch1, r0);
__ pop_call_clobbered_registers();
__ mov(r0, rscratch1);