diff -r 13588c901957 -r 9cf78a70fa4f src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp Thu Oct 17 20:27:44 2019 +0100 +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp Thu Oct 17 20:53:35 2019 +0100 @@ -24,7 +24,7 @@ #include "precompiled.hpp" #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" #include "gc/shenandoah/shenandoahForwarding.hpp" -#include "gc/shenandoah/shenandoahHeap.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahHeapRegion.hpp" #include "gc/shenandoah/shenandoahHeuristics.hpp" #include "gc/shenandoah/shenandoahRuntime.hpp" @@ -47,7 +47,7 @@ Register src, Register dst, Register count, RegSet saved_regs) { if (is_oop) { bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; - if (ShenandoahSATBBarrier && !dest_uninitialized && !ShenandoahHeap::heap()->heuristics()->can_do_traversal_gc()) { + if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) { Label done; @@ -57,27 +57,27 @@ // Is marking active? Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); __ ldrb(rscratch1, gc_state); - __ tbz(rscratch1, ShenandoahHeap::MARKING_BITPOS, done); + if (dest_uninitialized) { + __ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, done); + } else { + __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING); + __ tst(rscratch1, rscratch2); + __ br(Assembler::EQ, done); + } __ push(saved_regs, sp); - if (count == c_rarg0) { - if (dst == c_rarg1) { - // exactly backwards!! - __ mov(rscratch1, c_rarg0); - __ mov(c_rarg0, c_rarg1); - __ mov(c_rarg1, rscratch1); + if (UseCompressedOops) { + if (dest_uninitialized) { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_narrow_oop_entry), src, dst, count); } else { - __ mov(c_rarg1, count); - __ mov(c_rarg0, dst); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), src, dst, count); } } else { - __ mov(c_rarg0, dst); - __ mov(c_rarg1, count); - } - if (UseCompressedOops) { - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), 2); - } else { - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), 2); + if (dest_uninitialized) { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_oop_entry), src, dst, count); + } else { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), src, dst, count); + } } __ pop(saved_regs, sp); __ bind(done); @@ -85,31 +85,6 @@ } } -void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, - Register start, Register count, Register scratch, RegSet saved_regs) { - if (is_oop) { - Label done; - - // Avoid calling runtime if count == 0 - __ cbz(count, done); - - // Is updating references? - Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); - __ ldrb(rscratch1, gc_state); - __ tbz(rscratch1, ShenandoahHeap::UPDATEREFS_BITPOS, done); - - __ push(saved_regs, sp); - assert_different_registers(start, count, scratch); - assert_different_registers(c_rarg0, count); - __ mov(c_rarg0, start); - __ mov(c_rarg1, count); - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry), 2); - __ pop(saved_regs, sp); - - __ bind(done); - } -} - void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm, Register obj, Register pre_val, @@ -236,15 +211,20 @@ if (borrow_reg) { // No free registers available. Make one useful. tmp = rscratch1; + if (tmp == dst) { + tmp = rscratch2; + } __ push(RegSet::of(tmp), sp); } + assert_different_registers(tmp, dst); + Label done; __ ldr(tmp, Address(dst, oopDesc::mark_offset_in_bytes())); __ eon(tmp, tmp, zr); - __ ands(zr, tmp, markOopDesc::lock_mask_in_place); + __ ands(zr, tmp, markWord::lock_mask_in_place); __ br(Assembler::NE, done); - __ orr(tmp, tmp, markOopDesc::marked_value); + __ orr(tmp, tmp, markWord::marked_value); __ eon(dst, tmp, zr); __ bind(done); @@ -282,6 +262,40 @@ __ leave(); } +void ShenandoahBarrierSetAssembler::load_reference_barrier_native(MacroAssembler* masm, Register dst, Register tmp) { + if (!ShenandoahLoadRefBarrier) { + return; + } + + assert(dst != rscratch2, "need rscratch2"); + + Label is_null; + Label done; + + __ cbz(dst, is_null); + + __ enter(); + + Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); + __ ldrb(rscratch2, gc_state); + + // Check for heap in evacuation phase + __ tbz(rscratch2, ShenandoahHeap::EVACUATION_BITPOS, done); + + __ mov(rscratch2, dst); + __ push_call_clobbered_registers(); + __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native)); + __ mov(r0, rscratch2); + __ blr(lr); + __ mov(rscratch2, r0); + __ pop_call_clobbered_registers(); + __ mov(dst, rscratch2); + + __ bind(done); + __ leave(); + __ bind(is_null); +} + void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) { if (ShenandoahStoreValEnqueueBarrier) { // Save possibly live regs. @@ -308,16 +322,26 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread) { - bool on_oop = type == T_OBJECT || type == T_ARRAY; + bool on_oop = is_reference_type(type); + bool not_in_heap = (decorators & IN_NATIVE) != 0; bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; bool on_reference = on_weak || on_phantom; + bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0; BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); if (on_oop) { - load_reference_barrier(masm, dst, tmp1); - - if (ShenandoahKeepAliveBarrier && on_reference) { + if (not_in_heap) { + if (ShenandoahHeap::heap()->is_traversal_mode()) { + load_reference_barrier(masm, dst, tmp1); + keep_alive = true; + } else { + load_reference_barrier_native(masm, dst, tmp1); + } + } else { + load_reference_barrier(masm, dst, tmp1); + } + if (ShenandoahKeepAliveBarrier && on_reference && keep_alive) { __ enter(); satb_write_barrier_pre(masm /* masm */, noreg /* obj */, @@ -333,7 +357,7 @@ void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2) { - bool on_oop = type == T_OBJECT || type == T_ARRAY; + bool on_oop = is_reference_type(type); if (!on_oop) { BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2); return; @@ -371,6 +395,26 @@ } +void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env, + Register obj, Register tmp, Label& slowpath) { + Label done; + // Resolve jobject + BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath); + + // Check for null. + __ cbz(obj, done); + + assert(obj != rscratch2, "need rscratch2"); + Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); + __ ldrb(rscratch2, gc_state); + + // Check for heap in evacuation phase + __ tbnz(rscratch2, ShenandoahHeap::EVACUATION_BITPOS, slowpath); + + __ bind(done); +} + + void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val, bool acquire, bool release, bool weak, bool is_cae, Register result) { @@ -457,25 +501,47 @@ } void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) { + ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); + __ bind(*stub->entry()); Register obj = stub->obj()->as_register(); Register res = stub->result()->as_register(); + Register addr = stub->addr()->as_register_lo(); + Register tmp1 = stub->tmp1()->as_register(); + Register tmp2 = stub->tmp2()->as_register(); - Label done; - - __ bind(*stub->entry()); + assert(res == r0, "result must arrive in r0"); if (res != obj) { __ mov(res, obj); } + // Check for null. - if (stub->needs_null_check()) { - __ cbz(res, done); - } + __ cbz(res, *stub->continuation()); + + // Check for object in cset. + __ mov(tmp2, ShenandoahHeap::in_cset_fast_test_addr()); + __ lsr(tmp1, res, ShenandoahHeapRegion::region_size_bytes_shift_jint()); + __ ldrb(tmp2, Address(tmp2, tmp1)); + __ cbz(tmp2, *stub->continuation()); - load_reference_barrier_not_null(ce->masm(), res, rscratch1); + // Check if object is already forwarded. + Label slow_path; + __ ldr(tmp1, Address(res, oopDesc::mark_offset_in_bytes())); + __ eon(tmp1, tmp1, zr); + __ ands(zr, tmp1, markWord::lock_mask_in_place); + __ br(Assembler::NE, slow_path); - __ bind(done); + // Decode forwarded object. + __ orr(tmp1, tmp1, markWord::marked_value); + __ eon(res, tmp1, zr); + __ b(*stub->continuation()); + + __ bind(slow_path); + ce->store_parameter(res, 0); + ce->store_parameter(addr, 1); + __ far_call(RuntimeAddress(bs->load_reference_barrier_rt_code_blob()->code_begin())); + __ b(*stub->continuation()); } @@ -529,6 +595,26 @@ __ epilogue(); } +void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm) { + __ prologue("shenandoah_load_reference_barrier", false); + // arg0 : object to be resolved + + __ push_call_clobbered_registers(); + __ load_parameter(0, r0); + __ load_parameter(1, r1); + if (UseCompressedOops) { + __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup_narrow)); + } else { + __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup)); + } + __ blr(lr); + __ mov(rscratch1, r0); + __ pop_call_clobbered_registers(); + __ mov(r0, rscratch1); + + __ epilogue(); +} + #undef __ #endif // COMPILER1 @@ -563,17 +649,24 @@ __ ret(lr); __ bind(work); - __ mov(rscratch2, r0); - resolve_forward_pointer_not_null(cgen->assembler(), r0, rscratch1); - __ cmp(rscratch2, r0); - __ br(Assembler::NE, done); + Label slow_path; + __ ldr(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes())); + __ eon(rscratch1, rscratch1, zr); + __ ands(zr, rscratch1, markWord::lock_mask_in_place); + __ br(Assembler::NE, slow_path); + // Decode forwarded object. + __ orr(rscratch1, rscratch1, markWord::marked_value); + __ eon(r0, rscratch1, zr); + __ ret(lr); + + __ bind(slow_path); __ enter(); // required for proper stackwalking of RuntimeStub frame __ push_call_clobbered_registers(); - __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT)); - __ blrt(lr, 1, 0, MacroAssembler::ret_type_integral); + __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier)); + __ blr(lr); __ mov(rscratch1, r0); __ pop_call_clobbered_registers(); __ mov(r0, rscratch1);