# HG changeset patch # User rkennke # Date 1554221622 -7200 # Node ID cdc3bb0983a6acc29ea13e2aa1a91dc29fc13892 # Parent 61616f509ef8a4f936b2b226764d6229390c9ef8 8221751: Shenandoah: Improve SATB enqueueing Reviewed-by: shade diff -r 61616f509ef8 -r cdc3bb0983a6 src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp Tue Apr 02 08:19:36 2019 -0700 +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp Tue Apr 02 18:13:42 2019 +0200 @@ -46,7 +46,18 @@ Register addr, Register count, RegSet saved_regs) { if (is_oop) { bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; - if (!dest_uninitialized && !ShenandoahHeap::heap()->heuristics()->can_do_traversal_gc()) { + if (ShenandoahSATBBarrier && !dest_uninitialized && !ShenandoahHeap::heap()->heuristics()->can_do_traversal_gc()) { + + Label done; + + // Avoid calling runtime if count == 0 + __ cbz(count, done); + + // Is marking active? + Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); + __ ldrb(rscratch1, gc_state); + __ tbz(rscratch1, ShenandoahHeap::MARKING_BITPOS, done); + __ push(saved_regs, sp); if (count == c_rarg0) { if (addr == c_rarg1) { @@ -68,6 +79,7 @@ __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), 2); } __ pop(saved_regs, sp); + __ bind(done); } } } diff -r 61616f509ef8 -r cdc3bb0983a6 src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp Tue Apr 02 08:19:36 2019 -0700 +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp Tue Apr 02 18:13:42 2019 +0200 @@ -66,26 +66,22 @@ } #endif - if (!dest_uninitialized && !ShenandoahHeap::heap()->heuristics()->can_do_traversal_gc()) { + if (ShenandoahSATBBarrier && !dest_uninitialized && !ShenandoahHeap::heap()->heuristics()->can_do_traversal_gc()) { Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread); #ifndef _LP64 __ push(thread); __ get_thread(thread); #endif - Label filtered; - Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset())); - // Is marking active? - if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { - __ cmpl(in_progress, 0); - } else { - assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); - __ cmpb(in_progress, 0); - } + Label done; + // Short-circuit if count == 0. + __ testptr(count, count); + __ jcc(Assembler::zero, done); - NOT_LP64(__ pop(thread);) - - __ jcc(Assembler::equal, filtered); + // Avoid runtime call when not marking. + Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); + __ testb(gc_state, ShenandoahHeap::MARKING); + __ jcc(Assembler::zero, done); __ pusha(); // push registers #ifdef _LP64 @@ -111,7 +107,8 @@ dst, count); #endif __ popa(); - __ bind(filtered); + __ bind(done); + NOT_LP64(__ pop(thread);) } } diff -r 61616f509ef8 -r cdc3bb0983a6 src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp Tue Apr 02 08:19:36 2019 -0700 +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp Tue Apr 02 18:13:42 2019 +0200 @@ -132,12 +132,23 @@ template void ShenandoahBarrierSet::write_ref_array_pre_work(T* dst, size_t count) { shenandoah_assert_not_in_cset_loc_except(dst, _heap->cancelled_gc()); - if (ShenandoahSATBBarrier && _heap->is_concurrent_mark_in_progress()) { - T* elem_ptr = dst; - for (size_t i = 0; i < count; i++, elem_ptr++) { - T heap_oop = RawAccess<>::oop_load(elem_ptr); - if (!CompressedOops::is_null(heap_oop)) { - enqueue(CompressedOops::decode_not_null(heap_oop)); + assert(ShenandoahThreadLocalData::satb_mark_queue(Thread::current()).is_active(), "Shouldn't be here otherwise"); + assert(ShenandoahSATBBarrier, "Shouldn't be here otherwise"); + assert(count > 0, "Should have been filtered before"); + + Thread* thread = Thread::current(); + ShenandoahMarkingContext* ctx = _heap->marking_context(); + bool has_forwarded = _heap->has_forwarded_objects(); + T* elem_ptr = dst; + for (size_t i = 0; i < count; i++, elem_ptr++) { + T heap_oop = RawAccess<>::oop_load(elem_ptr); + if (!CompressedOops::is_null(heap_oop)) { + oop obj = CompressedOops::decode_not_null(heap_oop); + if (has_forwarded) { + obj = resolve_forwarded_not_null(obj); + } + if (!ctx->is_marked(obj)) { + ShenandoahThreadLocalData::satb_mark_queue(thread).enqueue_known_active(obj); } } } @@ -309,11 +320,9 @@ } oop ShenandoahBarrierSet::storeval_barrier(oop obj) { - if (ShenandoahStoreValEnqueueBarrier) { - if (!CompressedOops::is_null(obj)) { - obj = write_barrier(obj); - enqueue(obj); - } + if (ShenandoahStoreValEnqueueBarrier && !CompressedOops::is_null(obj) && _heap->is_concurrent_traversal_in_progress()) { + obj = write_barrier(obj); + enqueue(obj); } if (ShenandoahStoreValReadBarrier) { obj = resolve_forwarded(obj); @@ -329,14 +338,14 @@ void ShenandoahBarrierSet::enqueue(oop obj) { shenandoah_assert_not_forwarded_if(NULL, obj, _heap->is_concurrent_traversal_in_progress()); - if (!_satb_mark_queue_set.is_active()) return; + assert(_satb_mark_queue_set.is_active(), "only get here when SATB active"); // Filter marked objects before hitting the SATB queues. The same predicate would // be used by SATBMQ::filter to eliminate already marked objects downstream, but // filtering here helps to avoid wasteful SATB queueing work to begin with. if (!_heap->requires_marking(obj)) return; - ShenandoahThreadLocalData::satb_mark_queue(Thread::current()).enqueue(obj); + ShenandoahThreadLocalData::satb_mark_queue(Thread::current()).enqueue_known_active(obj); } void ShenandoahBarrierSet::on_thread_create(Thread* thread) { diff -r 61616f509ef8 -r cdc3bb0983a6 src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp Tue Apr 02 08:19:36 2019 -0700 +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp Tue Apr 02 18:13:42 2019 +0200 @@ -141,7 +141,7 @@ bool arraycopy_loop(T* src, T* dst, size_t length, Klass* bound, bool disjoint); template - bool arraycopy_element(T* cur_src, T* cur_dst, Klass* bound, Thread* thread); + bool arraycopy_element(T* cur_src, T* cur_dst, Klass* bound, Thread* const thread, ShenandoahMarkingContext* const ctx); public: // Callbacks for runtime accesses. diff -r 61616f509ef8 -r cdc3bb0983a6 src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp Tue Apr 02 08:19:36 2019 -0700 +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp Tue Apr 02 18:13:42 2019 +0200 @@ -28,6 +28,9 @@ #include "gc/shenandoah/shenandoahBarrierSet.hpp" #include "gc/shenandoah/shenandoahBrooksPointer.inline.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.hpp" +#include "gc/shenandoah/shenandoahMarkingContext.inline.hpp" +#include "gc/shenandoah/shenandoahThreadLocalData.hpp" bool ShenandoahBarrierSet::need_update_refs_barrier() { return _heap->is_update_refs_in_progress() || @@ -59,7 +62,8 @@ } while ((! oopDesc::equals_raw(compare_value, expected)) && oopDesc::equals_raw(resolve_forwarded(compare_value), resolve_forwarded(expected))); if (oopDesc::equals_raw(expected, compare_value)) { const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0; - if (keep_alive && ShenandoahSATBBarrier && !CompressedOops::is_null(compare_value)) { + if (keep_alive && ShenandoahSATBBarrier && !CompressedOops::is_null(compare_value) && + ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) { ShenandoahBarrierSet::barrier_set()->enqueue(compare_value); } } @@ -72,7 +76,8 @@ oop previous = Raw::oop_atomic_xchg(new_value, addr); if (ShenandoahSATBBarrier) { const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0; - if (keep_alive && !CompressedOops::is_null(previous)) { + if (keep_alive && !CompressedOops::is_null(previous) && + ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) { ShenandoahBarrierSet::barrier_set()->enqueue(previous); } } @@ -134,7 +139,7 @@ template bool ShenandoahBarrierSet::arraycopy_loop(T* src, T* dst, size_t length, Klass* bound, bool disjoint) { Thread* thread = Thread::current(); - + ShenandoahMarkingContext* ctx = _heap->marking_context(); ShenandoahEvacOOMScope oom_evac_scope; // We need to handle four cases: @@ -161,7 +166,7 @@ T* cur_dst = dst; T* src_end = src + length; for (; cur_src < src_end; cur_src++, cur_dst++) { - if (!arraycopy_element(cur_src, cur_dst, bound, thread)) { + if (!arraycopy_element(cur_src, cur_dst, bound, thread, ctx)) { return false; } } @@ -170,7 +175,7 @@ T* cur_src = src + length - 1; T* cur_dst = dst + length - 1; for (; cur_src >= src; cur_src--, cur_dst--) { - if (!arraycopy_element(cur_src, cur_dst, bound, thread)) { + if (!arraycopy_element(cur_src, cur_dst, bound, thread, ctx)) { return false; } } @@ -179,14 +184,26 @@ } template -bool ShenandoahBarrierSet::arraycopy_element(T* cur_src, T* cur_dst, Klass* bound, Thread* thread) { +bool ShenandoahBarrierSet::arraycopy_element(T* cur_src, T* cur_dst, Klass* bound, Thread* const thread, ShenandoahMarkingContext* const ctx) { T o = RawAccess<>::oop_load(cur_src); if (SATB) { + assert(ShenandoahThreadLocalData::satb_mark_queue(thread).is_active(), "Shouldn't be here otherwise"); T prev = RawAccess<>::oop_load(cur_dst); if (!CompressedOops::is_null(prev)) { oop prev_obj = CompressedOops::decode_not_null(prev); - enqueue(prev_obj); + switch (STOREVAL_MODE) { + case NONE: + break; + case READ_BARRIER: + case WRITE_BARRIER: + // The write-barrier case cannot really happen. It's traversal-only and traversal + // doesn't currently use SATB. And even if it did, it would not be fatal to just do the normal RB here. + prev_obj = ShenandoahBarrierSet::resolve_forwarded_not_null(prev_obj); + } + if (!ctx->is_marked(prev_obj)) { + ShenandoahThreadLocalData::satb_mark_queue(thread).enqueue_known_active(prev_obj); + } } } diff -r 61616f509ef8 -r cdc3bb0983a6 src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp Tue Apr 02 08:19:36 2019 -0700 +++ b/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp Tue Apr 02 18:13:42 2019 +0200 @@ -51,7 +51,8 @@ } shenandoah_assert_correct(NULL, orig); // store the original value that was in the field reference - ShenandoahThreadLocalData::satb_mark_queue(thread).enqueue(orig); + assert(ShenandoahThreadLocalData::satb_mark_queue(thread).is_active(), "Shouldn't be here otherwise"); + ShenandoahThreadLocalData::satb_mark_queue(thread).enqueue_known_active(orig); JRT_END JRT_LEAF(oopDesc*, ShenandoahRuntime::write_barrier_JRT(oopDesc* src))