# HG changeset patch # User zgu # Date 1574873577 18000 # Node ID 9186be5c78baaf0db58a5702a1deae73409293e9 # Parent 8b6cc0bb93d06ece61bb4140bcd5618d8045891e 8228720: Shenandoah: Implementation of concurrent class unloading Reviewed-by: rkennke diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp Wed Nov 27 06:36:41 2019 -0800 +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp Wed Nov 27 11:52:57 2019 -0500 @@ -395,6 +395,52 @@ __ block_comment("load_reference_barrier_native { "); } +#ifdef _LP64 +void ShenandoahBarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) { + // Use default version + BarrierSetAssembler::c2i_entry_barrier(masm); +} +#else +void ShenandoahBarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) { + BarrierSetNMethod* bs = BarrierSet::barrier_set()->barrier_set_nmethod(); + if (bs == NULL) { + return; + } + + Label bad_call; + __ cmpptr(rbx, 0); // rbx contains the incoming method for c2i adapters. + __ jcc(Assembler::equal, bad_call); + + Register tmp1 = rax; + Register tmp2 = rcx; + + __ push(tmp1); + __ push(tmp2); + + // Pointer chase to the method holder to find out if the method is concurrently unloading. + Label method_live; + __ load_method_holder_cld(tmp1, rbx); + + // Is it a strong CLD? + __ cmpl(Address(tmp1, ClassLoaderData::keep_alive_offset()), 0); + __ jcc(Assembler::greater, method_live); + + // Is it a weak but alive CLD? + __ movptr(tmp1, Address(tmp1, ClassLoaderData::holder_offset())); + __ resolve_weak_handle(tmp1, tmp2); + __ cmpptr(tmp1, 0); + __ jcc(Assembler::notEqual, method_live); + __ pop(tmp2); + __ pop(tmp1); + + __ bind(bad_call); + __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); + __ bind(method_live); + __ pop(tmp2); + __ pop(tmp1); +} +#endif + void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) { if (ShenandoahStoreValEnqueueBarrier) { storeval_barrier_impl(masm, dst, tmp); @@ -512,8 +558,11 @@ // 3: apply keep-alive barrier if needed if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) { __ push_IU_state(); - const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread); + Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread); assert_different_registers(dst, tmp1, tmp_thread); + if (!thread->is_valid()) { + thread = rdx; + } NOT_LP64(__ get_thread(thread)); // Generate the SATB pre-barrier code to log the value of // the referent field in an SATB buffer. diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp Wed Nov 27 06:36:41 2019 -0800 +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp Wed Nov 27 11:52:57 2019 -0500 @@ -86,6 +86,7 @@ Address dst, Register val, Register tmp1, Register tmp2); virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env, Register obj, Register tmp, Label& slowpath); + virtual void c2i_entry_barrier(MacroAssembler* masm); virtual void barrier_stubs_init(); diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp Wed Nov 27 06:36:41 2019 -0800 +++ b/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp Wed Nov 27 11:52:57 2019 -0500 @@ -171,11 +171,7 @@ } // If class unloading is disabled, no unloading for concurrent cycles as well. - // If class unloading is enabled, users should opt-in for unloading during - // concurrent cycles. - if (!ClassUnloading || !FLAG_IS_CMDLINE(ClassUnloadingWithConcurrentMark)) { - log_info(gc)("Consider -XX:+ClassUnloadingWithConcurrentMark if large pause times " - "are observed on class-unloading sensitive workloads"); + if (!ClassUnloading) { FLAG_SET_DEFAULT(ClassUnloadingWithConcurrentMark, false); } diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp Wed Nov 27 06:36:41 2019 -0800 +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp Wed Nov 27 11:52:57 2019 -0500 @@ -26,6 +26,7 @@ #include "gc/shenandoah/shenandoahBarrierSet.hpp" #include "gc/shenandoah/shenandoahBarrierSetClone.inline.hpp" #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" +#include "gc/shenandoah/shenandoahBarrierSetNMethod.hpp" #include "gc/shenandoah/shenandoahCollectorPolicy.hpp" #include "gc/shenandoah/shenandoahConcurrentRoots.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" @@ -43,11 +44,19 @@ class ShenandoahBarrierSetC1; class ShenandoahBarrierSetC2; +static BarrierSetNMethod* make_barrier_set_nmethod(ShenandoahHeap* heap) { + // NMethod barriers are only used when concurrent nmethod unloading is enabled + if (!ShenandoahConcurrentRoots::can_do_concurrent_class_unloading()) { + return NULL; + } + return new ShenandoahBarrierSetNMethod(heap); +} + ShenandoahBarrierSet::ShenandoahBarrierSet(ShenandoahHeap* heap) : BarrierSet(make_barrier_set_assembler(), make_barrier_set_c1(), make_barrier_set_c2(), - NULL /* barrier_set_nmethod */, + make_barrier_set_nmethod(heap), BarrierSet::FakeRtti(BarrierSet::ShenandoahBarrierSet)), _heap(heap), _satb_mark_queue_buffer_allocator("SATB Buffer Allocator", ShenandoahSATBBufferSize), diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahBarrierSetNMethod.cpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetNMethod.cpp Wed Nov 27 11:52:57 2019 -0500 @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2019, Red Hat, Inc. All rights reserved. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "gc/shenandoah/shenandoahBarrierSetNMethod.hpp" +#include "gc/shenandoah/shenandoahClosures.inline.hpp" +#include "gc/shenandoah/shenandoahCodeRoots.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahLock.hpp" +#include "gc/shenandoah/shenandoahNMethod.inline.hpp" +#include "gc/shenandoah/shenandoahThreadLocalData.hpp" +#include "memory/iterator.hpp" +#include "memory/resourceArea.hpp" + +bool ShenandoahBarrierSetNMethod::nmethod_entry_barrier(nmethod* nm) { + ShenandoahReentrantLock* lock = ShenandoahNMethod::lock_for_nmethod(nm); + assert(lock != NULL, "Must be"); + ShenandoahReentrantLocker locker(lock); + + if (!is_armed(nm)) { + // Some other thread got here first and healed the oops + // and disarmed the nmethod. + return true; + } + + if (nm->is_unloading()) { + // We don't need to take the lock when unlinking nmethods from + // the Method, because it is only concurrently unlinked by + // the entry barrier, which acquires the per nmethod lock. + nm->unlink_from_method(); + + // We can end up calling nmethods that are unloading + // since we clear compiled ICs lazily. Returning false + // will re-resovle the call and update the compiled IC. + return false; + } + + // Heal oops and disarm + ShenandoahEvacOOMScope scope; + ShenandoahNMethod::heal_nmethod(nm); + ShenandoahNMethod::disarm_nmethod(nm); + return true; +} + +int ShenandoahBarrierSetNMethod::disarmed_value() const { + return ShenandoahCodeRoots::disarmed_value(); +} + +ByteSize ShenandoahBarrierSetNMethod::thread_disarmed_offset() const { + return ShenandoahThreadLocalData::disarmed_value_offset(); +} + +int* ShenandoahBarrierSetNMethod::disarmed_value_address() const { + return ShenandoahCodeRoots::disarmed_value_address(); +} diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahBarrierSetNMethod.hpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetNMethod.hpp Wed Nov 27 11:52:57 2019 -0500 @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2019, Red Hat, Inc. All rights reserved. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHBARRIERSETNMETHOD_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHBARRIERSETNMETHOD_HPP + +#include "gc/shared/barrierSetNMethod.hpp" +#include "memory/allocation.hpp" + +class nmethod; +class ShenandoahHeap; + +class ShenandoahBarrierSetNMethod : public BarrierSetNMethod { +private: + ShenandoahHeap* _heap; + +protected: + virtual int disarmed_value() const; + virtual bool nmethod_entry_barrier(nmethod* nm); + +public: + ShenandoahBarrierSetNMethod(ShenandoahHeap* heap) : _heap(heap) { + } + + virtual ByteSize thread_disarmed_offset() const; + virtual int* disarmed_value_address() const; +}; + +#endif // SHARE_GC_SHENANDOAH_SHENANDOAHBARRIERSETNMETHOD_HPP diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahClosures.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahClosures.hpp Wed Nov 27 06:36:41 2019 -0800 +++ b/src/hotspot/share/gc/shenandoah/shenandoahClosures.hpp Wed Nov 27 11:52:57 2019 -0500 @@ -104,6 +104,15 @@ inline void do_oop(narrowOop* p); }; +class ShenandoahCodeBlobAndDisarmClosure: public CodeBlobToOopClosure { +private: + BarrierSetNMethod* const _bs; + +public: + inline ShenandoahCodeBlobAndDisarmClosure(OopClosure* cl); + inline void do_code_blob(CodeBlob* cb); +}; + #ifdef ASSERT class ShenandoahAssertNotForwardedClosure : public OopClosure { private: diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp Wed Nov 27 06:36:41 2019 -0800 +++ b/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp Wed Nov 27 11:52:57 2019 -0500 @@ -23,9 +23,11 @@ #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHCLOSURES_INLINE_HPP #define SHARE_GC_SHENANDOAH_SHENANDOAHCLOSURES_INLINE_HPP +#include "gc/shared/barrierSetNMethod.hpp" #include "gc/shenandoah/shenandoahAsserts.hpp" #include "gc/shenandoah/shenandoahClosures.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahNMethod.inline.hpp" #include "gc/shenandoah/shenandoahTraversalGC.hpp" #include "oops/compressedOops.inline.hpp" #include "runtime/atomic.hpp" @@ -157,6 +159,20 @@ ShouldNotReachHere(); } +ShenandoahCodeBlobAndDisarmClosure::ShenandoahCodeBlobAndDisarmClosure(OopClosure* cl) : + CodeBlobToOopClosure(cl, true /* fix_relocations */), + _bs(BarrierSet::barrier_set()->barrier_set_nmethod()) { +} + +void ShenandoahCodeBlobAndDisarmClosure::do_code_blob(CodeBlob* cb) { + nmethod* const nm = cb->as_nmethod_or_null(); + if (nm != NULL && nm->oops_do_try_claim()) { + assert(!ShenandoahNMethod::gc_data(nm)->is_unregistered(), "Should not be here"); + CodeBlobToOopClosure::do_code_blob(cb); + _bs->disarm(nm); + } +} + #ifdef ASSERT template void ShenandoahAssertNotForwardedClosure::do_oop_work(T* p) { diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp Wed Nov 27 06:36:41 2019 -0800 +++ b/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp Wed Nov 27 11:52:57 2019 -0500 @@ -23,9 +23,11 @@ #include "precompiled.hpp" #include "code/codeCache.hpp" +#include "code/icBuffer.hpp" #include "code/nmethod.hpp" +#include "gc/shenandoah/shenandoahCodeRoots.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" -#include "gc/shenandoah/shenandoahCodeRoots.hpp" +#include "gc/shenandoah/shenandoahNMethod.inline.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" #include "memory/resourceArea.hpp" #include "memory/universe.hpp" @@ -98,69 +100,45 @@ _finished = true; } -class ShenandoahNMethodOopDetector : public OopClosure { -private: - ResourceMark rm; // For growable array allocation below. - GrowableArray _oops; - -public: - ShenandoahNMethodOopDetector() : _oops(10) {}; - - void do_oop(oop* o) { - _oops.append(o); - } - void do_oop(narrowOop* o) { - fatal("NMethods should not have compressed oops embedded."); - } - - GrowableArray* oops() { - return &_oops; - } - - bool has_oops() { - return !_oops.is_empty(); - } -}; - -GrowableArray* ShenandoahCodeRoots::_recorded_nms; -ShenandoahLock ShenandoahCodeRoots::_recorded_nms_lock; +ShenandoahNMethodTable* ShenandoahCodeRoots::_nmethod_table; +int ShenandoahCodeRoots::_disarmed_value = 1; void ShenandoahCodeRoots::initialize() { - _recorded_nms = new (ResourceObj::C_HEAP, mtGC) GrowableArray(100, true, mtGC); + _nmethod_table = new ShenandoahNMethodTable(); } -void ShenandoahCodeRoots::add_nmethod(nmethod* nm) { +void ShenandoahCodeRoots::register_nmethod(nmethod* nm) { switch (ShenandoahCodeRootsStyle) { case 0: case 1: break; case 2: { assert_locked_or_safepoint(CodeCache_lock); - ShenandoahLocker locker(CodeCache_lock->owned_by_self() ? NULL : &_recorded_nms_lock); - - ShenandoahNMethodOopDetector detector; - nm->oops_do(&detector); - - if (detector.has_oops()) { - ShenandoahNMethod* nmr = new ShenandoahNMethod(nm, detector.oops()); - nmr->assert_alive_and_correct(); - int idx = _recorded_nms->find(nm, ShenandoahNMethod::find_with_nmethod); - if (idx != -1) { - ShenandoahNMethod* old = _recorded_nms->at(idx); - _recorded_nms->at_put(idx, nmr); - delete old; - } else { - _recorded_nms->append(nmr); - } - } + _nmethod_table->register_nmethod(nm); break; } default: ShouldNotReachHere(); } -}; +} -void ShenandoahCodeRoots::remove_nmethod(nmethod* nm) { +void ShenandoahCodeRoots::unregister_nmethod(nmethod* nm) { + switch (ShenandoahCodeRootsStyle) { + case 0: + case 1: { + break; + } + case 2: { + assert_locked_or_safepoint(CodeCache_lock); + _nmethod_table->unregister_nmethod(nm); + break; + } + default: + ShouldNotReachHere(); + } +} + +void ShenandoahCodeRoots::flush_nmethod(nmethod* nm) { switch (ShenandoahCodeRootsStyle) { case 0: case 1: { @@ -168,19 +146,7 @@ } case 2: { assert_locked_or_safepoint(CodeCache_lock); - ShenandoahLocker locker(CodeCache_lock->owned_by_self() ? NULL : &_recorded_nms_lock); - - ShenandoahNMethodOopDetector detector; - nm->oops_do(&detector, /* allow_dead = */ true); - - if (detector.has_oops()) { - int idx = _recorded_nms->find(nm, ShenandoahNMethod::find_with_nmethod); - assert(idx != -1, "nmethod " PTR_FORMAT " should be registered", p2i(nm)); - ShenandoahNMethod* old = _recorded_nms->at(idx); - old->assert_same_oops(detector.oops()); - _recorded_nms->delete_at(idx); - delete old; - } + _nmethod_table->flush_nmethod(nm); break; } default: @@ -188,10 +154,186 @@ } } +void ShenandoahCodeRoots::prepare_concurrent_unloading() { + assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint"); + _disarmed_value ++; + // 0 is reserved for new nmethod + if (_disarmed_value == 0) { + _disarmed_value = 1; + } + + JavaThreadIteratorWithHandle jtiwh; + for (JavaThread *thr = jtiwh.next(); thr != NULL; thr = jtiwh.next()) { + ShenandoahThreadLocalData::set_disarmed_value(thr, _disarmed_value); + } +} + +class ShenandoahNMethodUnlinkClosure : public NMethodClosure { +private: + bool _unloading_occurred; + volatile bool _failed; + ShenandoahHeap* _heap; + + void set_failed() { + Atomic::store(&_failed, true); + } + + void unlink(nmethod* nm) { + // Unlinking of the dependencies must happen before the + // handshake separating unlink and purge. + nm->flush_dependencies(false /* delete_immediately */); + + // unlink_from_method will take the CompiledMethod_lock. + // In this case we don't strictly need it when unlinking nmethods from + // the Method, because it is only concurrently unlinked by + // the entry barrier, which acquires the per nmethod lock. + nm->unlink_from_method(); + + if (nm->is_osr_method()) { + // Invalidate the osr nmethod only once + nm->invalidate_osr_method(); + } + } +public: + ShenandoahNMethodUnlinkClosure(bool unloading_occurred) : + _unloading_occurred(unloading_occurred), + _failed(false), + _heap(ShenandoahHeap::heap()) {} + + virtual void do_nmethod(nmethod* nm) { + if (failed()) { + return; + } + + ShenandoahNMethod* nm_data = ShenandoahNMethod::gc_data(nm); + assert(!nm_data->is_unregistered(), "Should not see unregistered entry"); + + if (!nm->is_alive()) { + return; + } + + if (nm->is_unloading()) { + ShenandoahReentrantLocker locker(nm_data->lock()); + unlink(nm); + return; + } + + ShenandoahReentrantLocker locker(nm_data->lock()); + + // Heal oops and disarm + ShenandoahEvacOOMScope scope; + ShenandoahNMethod::heal_nmethod(nm); + ShenandoahNMethod::disarm_nmethod(nm); + + // Clear compiled ICs and exception caches + if (!nm->unload_nmethod_caches(_unloading_occurred)) { + set_failed(); + } + } + + bool failed() const { + return Atomic::load(&_failed); + } +}; + +class ShenandoahUnlinkTask : public AbstractGangTask { +private: + ShenandoahNMethodUnlinkClosure _cl; + ICRefillVerifier* _verifier; + ShenandoahConcurrentNMethodIterator _iterator; + +public: + ShenandoahUnlinkTask(bool unloading_occurred, ICRefillVerifier* verifier) : + AbstractGangTask("ShenandoahNMethodUnlinkTask"), + _cl(unloading_occurred), + _verifier(verifier), + _iterator(ShenandoahCodeRoots::table()) { + MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag); + _iterator.nmethods_do_begin(); + } + + ~ShenandoahUnlinkTask() { + MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag); + _iterator.nmethods_do_end(); + } + + virtual void work(uint worker_id) { + ICRefillVerifierMark mark(_verifier); + _iterator.nmethods_do(&_cl); + } + + bool success() const { + return !_cl.failed(); + } +}; + +void ShenandoahCodeRoots::unlink(WorkGang* workers, bool unloading_occurred) { + assert(ShenandoahConcurrentRoots::should_do_concurrent_class_unloading(), + "Only when running concurrent class unloading"); + + for (;;) { + ICRefillVerifier verifier; + + { + ShenandoahUnlinkTask task(unloading_occurred, &verifier); + workers->run_task(&task); + if (task.success()) { + return; + } + } + + // Cleaning failed because we ran out of transitional IC stubs, + // so we have to refill and try again. Refilling requires taking + // a safepoint, so we temporarily leave the suspendible thread set. + SuspendibleThreadSetLeaver sts; + InlineCacheBuffer::refill_ic_stubs(); + } +} + +class ShenandoahNMethodPurgeClosure : public NMethodClosure { +public: + virtual void do_nmethod(nmethod* nm) { + if (nm->is_alive() && nm->is_unloading()) { + nm->make_unloaded(); + } + } +}; + +class ShenandoahNMethodPurgeTask : public AbstractGangTask { +private: + ShenandoahNMethodPurgeClosure _cl; + ShenandoahConcurrentNMethodIterator _iterator; + +public: + ShenandoahNMethodPurgeTask() : + AbstractGangTask("ShenandoahNMethodPurgeTask"), + _cl(), + _iterator(ShenandoahCodeRoots::table()) { + MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag); + _iterator.nmethods_do_begin(); + } + + ~ShenandoahNMethodPurgeTask() { + MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag); + _iterator.nmethods_do_end(); + } + + virtual void work(uint worker_id) { + _iterator.nmethods_do(&_cl); + } +}; + +void ShenandoahCodeRoots::purge(WorkGang* workers) { + assert(ShenandoahConcurrentRoots::should_do_concurrent_class_unloading(), + "Only when running concurrent class unloading"); + + ShenandoahNMethodPurgeTask task; + workers->run_task(&task); +} + ShenandoahCodeRootsIterator::ShenandoahCodeRootsIterator() : - _heap(ShenandoahHeap::heap()), _par_iterator(CodeCache::heaps()), - _claimed(0) { + _table_snapshot(NULL) { assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint"); assert(!Thread::current()->is_Worker_thread(), "Should not be acquired by workers"); switch (ShenandoahCodeRootsStyle) { @@ -202,6 +344,7 @@ } case 2: { CodeCache_lock->lock_without_safepoint_check(); + _table_snapshot = ShenandoahCodeRoots::table()->snapshot_for_iteration(); break; } default: @@ -217,6 +360,8 @@ break; } case 2: { + ShenandoahCodeRoots::table()->finish_iteration(_table_snapshot); + _table_snapshot = NULL; CodeCache_lock->unlock(); break; } @@ -258,77 +403,7 @@ template void ShenandoahCodeRootsIterator::fast_parallel_blobs_do(CodeBlobClosure *f) { assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint"); - - size_t stride = 256; // educated guess - - GrowableArray* list = ShenandoahCodeRoots::_recorded_nms; - - size_t max = (size_t)list->length(); - while (_claimed < max) { - size_t cur = Atomic::add(&_claimed, stride) - stride; - size_t start = cur; - size_t end = MIN2(cur + stride, max); - if (start >= max) break; - - for (size_t idx = start; idx < end; idx++) { - ShenandoahNMethod* nmr = list->at((int) idx); - nmr->assert_alive_and_correct(); - - if (CSET_FILTER && !nmr->has_cset_oops(_heap)) { - continue; - } - - f->do_code_blob(nmr->nm()); - } - } -} - -ShenandoahNMethod::ShenandoahNMethod(nmethod* nm, GrowableArray* oops) { - _nm = nm; - _oops = NEW_C_HEAP_ARRAY(oop*, oops->length(), mtGC); - _oops_count = oops->length(); - for (int c = 0; c < _oops_count; c++) { - _oops[c] = oops->at(c); - } + assert(_table_snapshot != NULL, "Sanity"); + _table_snapshot->parallel_blobs_do(f); } -ShenandoahNMethod::~ShenandoahNMethod() { - if (_oops != NULL) { - FREE_C_HEAP_ARRAY(oop*, _oops); - } -} - -bool ShenandoahNMethod::has_cset_oops(ShenandoahHeap *heap) { - for (int c = 0; c < _oops_count; c++) { - oop o = RawAccess<>::oop_load(_oops[c]); - if (heap->in_collection_set(o)) { - return true; - } - } - return false; -} - -#ifdef ASSERT -void ShenandoahNMethod::assert_alive_and_correct() { - assert(_nm->is_alive(), "only alive nmethods here"); - assert(_oops_count > 0, "should have filtered nmethods without oops before"); - ShenandoahHeap* heap = ShenandoahHeap::heap(); - for (int c = 0; c < _oops_count; c++) { - oop *loc = _oops[c]; - assert(_nm->code_contains((address) loc) || _nm->oops_contains(loc), "nmethod should contain the oop*"); - oop o = RawAccess<>::oop_load(loc); - shenandoah_assert_correct_except(loc, o, - o == NULL || - heap->is_full_gc_move_in_progress() || - (VMThread::vm_operation() != NULL) && (VMThread::vm_operation()->type() == VM_Operation::VMOp_HeapWalkOperation) - ); - } -} - -void ShenandoahNMethod::assert_same_oops(GrowableArray* oops) { - assert(_oops_count == oops->length(), "should have the same number of oop*"); - for (int c = 0; c < _oops_count; c++) { - assert(_oops[c] == oops->at(c), "should be the same oop*"); - } -} -#endif diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.hpp Wed Nov 27 06:36:41 2019 -0800 +++ b/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.hpp Wed Nov 27 11:52:57 2019 -0500 @@ -27,6 +27,7 @@ #include "code/codeCache.hpp" #include "gc/shenandoah/shenandoahSharedVariables.hpp" #include "gc/shenandoah/shenandoahLock.hpp" +#include "gc/shenandoah/shenandoahNMethod.hpp" #include "memory/allocation.hpp" #include "memory/iterator.hpp" @@ -62,43 +63,13 @@ void parallel_blobs_do(CodeBlobClosure* f); }; -// ShenandoahNMethod tuple records the internal locations of oop slots within the nmethod. -// This allows us to quickly scan the oops without doing the nmethod-internal scans, that -// sometimes involves parsing the machine code. Note it does not record the oops themselves, -// because it would then require handling these tuples as the new class of roots. -class ShenandoahNMethod : public CHeapObj { -private: - nmethod* _nm; - oop** _oops; - int _oops_count; - -public: - ShenandoahNMethod(nmethod *nm, GrowableArray* oops); - ~ShenandoahNMethod(); - - nmethod* nm() { - return _nm; - } - - bool has_cset_oops(ShenandoahHeap* heap); - - void assert_alive_and_correct() NOT_DEBUG_RETURN; - void assert_same_oops(GrowableArray* oops) NOT_DEBUG_RETURN; - - static bool find_with_nmethod(void* nm, ShenandoahNMethod* other) { - return other->_nm == nm; - } -}; - class ShenandoahCodeRootsIterator { friend class ShenandoahCodeRoots; protected: - ShenandoahHeap* _heap; ShenandoahParallelCodeCacheIterator _par_iterator; ShenandoahSharedFlag _seq_claimed; - DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile size_t)); - volatile size_t _claimed; - DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0); + ShenandoahNMethodTableSnapshot* _table_snapshot; + protected: ShenandoahCodeRootsIterator(); ~ShenandoahCodeRootsIterator(); @@ -128,12 +99,24 @@ public: static void initialize(); - static void add_nmethod(nmethod* nm); - static void remove_nmethod(nmethod* nm); + static void register_nmethod(nmethod* nm); + static void unregister_nmethod(nmethod* nm); + static void flush_nmethod(nmethod* nm); + + static ShenandoahNMethodTable* table() { + return _nmethod_table; + } + + // Concurrent nmethod unloading support + static void unlink(WorkGang* workers, bool unloading_occurred); + static void purge(WorkGang* workers); + static void prepare_concurrent_unloading(); + static int disarmed_value() { return _disarmed_value; } + static int* disarmed_value_address() { return &_disarmed_value; } private: - static GrowableArray* _recorded_nms; - static ShenandoahLock _recorded_nms_lock; + static ShenandoahNMethodTable* _nmethod_table; + static int _disarmed_value; }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHCODEROOTS_HPP diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahConcurrentRoots.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentRoots.cpp Wed Nov 27 06:36:41 2019 -0800 +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentRoots.cpp Wed Nov 27 11:52:57 2019 -0500 @@ -32,9 +32,21 @@ } bool ShenandoahConcurrentRoots::should_do_concurrent_roots() { - ShenandoahHeap* const heap = ShenandoahHeap::heap(); - bool stw_gc_in_progress = heap->is_full_gc_in_progress() || - heap->is_degenerated_gc_in_progress(); return can_do_concurrent_roots() && - !stw_gc_in_progress; + !ShenandoahHeap::heap()->is_stw_gc_in_progress(); } + +bool ShenandoahConcurrentRoots::can_do_concurrent_class_unloading() { +#if defined(X86) && !defined(SOLARIS) + return ShenandoahCodeRootsStyle == 2 && + ClassUnloading && + strcmp(ShenandoahGCMode, "traversal") != 0; +#else + return false; +#endif +} + +bool ShenandoahConcurrentRoots::should_do_concurrent_class_unloading() { + return can_do_concurrent_class_unloading() && + !ShenandoahHeap::heap()->is_stw_gc_in_progress(); +} diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahConcurrentRoots.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentRoots.hpp Wed Nov 27 06:36:41 2019 -0800 +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentRoots.hpp Wed Nov 27 11:52:57 2019 -0500 @@ -32,6 +32,11 @@ static bool can_do_concurrent_roots(); // If current GC cycle can process roots concurrently static bool should_do_concurrent_roots(); + + // If GC settings allow concurrent class unloading + static bool can_do_concurrent_class_unloading(); + // If current GC cycle can unload classes concurrently + static bool should_do_concurrent_class_unloading(); }; diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp Wed Nov 27 06:36:41 2019 -0800 +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp Wed Nov 27 11:52:57 2019 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2018, Red Hat, Inc. All rights reserved. + * Copyright (c) 2016, 2019, Red Hat, Inc. All rights reserved. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as @@ -149,6 +149,11 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, ShenandoahAllocRequest& req, bool& in_new_region) { assert (!has_no_alloc_capacity(r), "Performance: should avoid full regions on this path: " SIZE_FORMAT, r->region_number()); + if (_heap->is_concurrent_root_in_progress() && + r->is_trash()) { + return NULL; + } + try_recycle_trashed(r); in_new_region = r->is_empty(); diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp Wed Nov 27 06:36:41 2019 -0800 +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp Wed Nov 27 11:52:57 2019 -0500 @@ -1076,7 +1076,8 @@ // Include concurrent roots if current cycle can not process those roots concurrently ShenandoahRootEvacuator rp(workers()->active_workers(), ShenandoahPhaseTimings::init_evac, - !ShenandoahConcurrentRoots::should_do_concurrent_roots()); + !ShenandoahConcurrentRoots::should_do_concurrent_roots(), + !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()); ShenandoahEvacuateUpdateRootsTask roots_task(&rp); workers()->run_task(&roots_task); } @@ -1548,6 +1549,8 @@ set_has_forwarded_objects(true); if (!is_degenerated_gc_in_progress()) { + prepare_concurrent_roots(); + prepare_concurrent_unloading(); evacuate_and_update_roots(); } @@ -1556,13 +1559,16 @@ } if (ShenandoahVerify) { + ShenandoahRootVerifier::RootTypes types = ShenandoahRootVerifier::None; if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) { - ShenandoahRootVerifier::RootTypes types = ShenandoahRootVerifier::combine(ShenandoahRootVerifier::JNIHandleRoots, ShenandoahRootVerifier::WeakRoots); + types = ShenandoahRootVerifier::combine(ShenandoahRootVerifier::JNIHandleRoots, ShenandoahRootVerifier::WeakRoots); types = ShenandoahRootVerifier::combine(types, ShenandoahRootVerifier::CLDGRoots); - verifier()->verify_roots_no_forwarded_except(types); - } else { - verifier()->verify_roots_no_forwarded(); } + + if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) { + types = ShenandoahRootVerifier::combine(types, ShenandoahRootVerifier::CodeRoots); + } + verifier()->verify_roots_no_forwarded_except(types); verifier()->verify_during_evacuation(); } } else { @@ -1658,11 +1664,18 @@ }; void ShenandoahHeap::op_roots() { - if (is_evacuation_in_progress() && - ShenandoahConcurrentRoots::should_do_concurrent_roots()) { - ShenandoahConcurrentRootsEvacUpdateTask task; - workers()->run_task(&task); + if (is_evacuation_in_progress()) { + if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) { + _unloader.unload(); + } + + if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) { + ShenandoahConcurrentRootsEvacUpdateTask task; + workers()->run_task(&task); + } } + + set_concurrent_root_in_progress(false); } void ShenandoahHeap::op_reset() { @@ -1920,6 +1933,15 @@ set_gc_state_mask(EVACUATION, in_progress); } +void ShenandoahHeap::set_concurrent_root_in_progress(bool in_progress) { + assert(ShenandoahConcurrentRoots::can_do_concurrent_roots(), "Why set the flag?"); + if (in_progress) { + _concurrent_root_in_progress.set(); + } else { + _concurrent_root_in_progress.unset(); + } +} + void ShenandoahHeap::ref_processing_init() { assert(_max_workers > 0, "Sanity"); @@ -2028,10 +2050,10 @@ MetaspaceUtils::verify_metrics(); } -// Process leftover weak oops: update them, if needed or assert they do not -// need updating otherwise. -// Weak processor API requires us to visit the oops, even if we are not doing -// anything to them. +// Weak roots are either pre-evacuated (final mark) or updated (final updaterefs), +// so they should not have forwarded oops. +// However, we do need to "null" dead oops in the roots, if can not be done +// in concurrent cycles. void ShenandoahHeap::stw_process_weak_roots(bool full_gc) { ShenandoahGCPhase root_phase(full_gc ? ShenandoahPhaseTimings::full_gc_purge : @@ -2073,7 +2095,9 @@ void ShenandoahHeap::parallel_cleaning(bool full_gc) { assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint"); stw_process_weak_roots(full_gc); - stw_unload_classes(full_gc); + if (!ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) { + stw_unload_classes(full_gc); + } } void ShenandoahHeap::set_has_forwarded_objects(bool cond) { @@ -2141,11 +2165,15 @@ } void ShenandoahHeap::register_nmethod(nmethod* nm) { - ShenandoahCodeRoots::add_nmethod(nm); + ShenandoahCodeRoots::register_nmethod(nm); } void ShenandoahHeap::unregister_nmethod(nmethod* nm) { - ShenandoahCodeRoots::remove_nmethod(nm); + ShenandoahCodeRoots::unregister_nmethod(nm); +} + +void ShenandoahHeap::flush_nmethod(nmethod* nm) { + ShenandoahCodeRoots::flush_nmethod(nm); } oop ShenandoahHeap::pin_object(JavaThread* thr, oop o) { @@ -2192,6 +2220,28 @@ return _gc_timer; } +void ShenandoahHeap::prepare_concurrent_roots() { + assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint"); + if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) { + set_concurrent_root_in_progress(true); + } +} + +void ShenandoahHeap::prepare_concurrent_unloading() { + assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint"); + if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) { + ShenandoahCodeRoots::prepare_concurrent_unloading(); + _unloader.prepare(); + } +} + +void ShenandoahHeap::finish_concurrent_unloading() { + assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint"); + if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) { + _unloader.finish(); + } +} + #ifdef ASSERT void ShenandoahHeap::assert_gc_workers(uint nworkers) { assert(nworkers > 0 && nworkers <= max_workers(), "Sanity"); @@ -2315,6 +2365,8 @@ void ShenandoahHeap::op_final_updaterefs() { assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint"); + finish_concurrent_unloading(); + // Check if there is left-over work, and finish it if (_update_refs_iterator.has_next()) { ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_finish_work); @@ -2332,7 +2384,7 @@ assert(!cancelled_gc(), "Should have been done right before"); if (ShenandoahVerify && !is_degenerated_gc_in_progress()) { - verifier()->verify_roots_no_forwarded_except(ShenandoahRootVerifier::ThreadRoots); + verifier()->verify_roots_in_to_space_except(ShenandoahRootVerifier::ThreadRoots); } if (is_degenerated_gc_in_progress()) { diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp Wed Nov 27 06:36:41 2019 -0800 +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp Wed Nov 27 11:52:57 2019 -0500 @@ -32,6 +32,7 @@ #include "gc/shenandoah/shenandoahLock.hpp" #include "gc/shenandoah/shenandoahEvacOOMHandler.hpp" #include "gc/shenandoah/shenandoahSharedVariables.hpp" +#include "gc/shenandoah/shenandoahUnload.hpp" #include "services/memoryManager.hpp" class ConcurrentGCTimer; @@ -271,6 +272,7 @@ ShenandoahSharedFlag _full_gc_in_progress; ShenandoahSharedFlag _full_gc_move_in_progress; ShenandoahSharedFlag _progress_last_gc; + ShenandoahSharedFlag _concurrent_root_in_progress; void set_gc_state_all_threads(char state); void set_gc_state_mask(uint mask, bool value); @@ -287,6 +289,7 @@ void set_full_gc_move_in_progress(bool in_progress); void set_concurrent_traversal_in_progress(bool in_progress); void set_has_forwarded_objects(bool cond); + void set_concurrent_root_in_progress(bool cond); inline bool is_stable() const; inline bool is_idle() const; @@ -299,6 +302,8 @@ inline bool is_concurrent_traversal_in_progress() const; inline bool has_forwarded_objects() const; inline bool is_gc_in_progress_mask(uint mask) const; + inline bool is_stw_gc_in_progress() const; + inline bool is_concurrent_root_in_progress() const; // ---------- GC cancellation and degeneration machinery // @@ -511,6 +516,7 @@ // private: ShenandoahSharedFlag _unload_classes; + ShenandoahUnload _unloader; public: void set_unload_classes(bool uc); @@ -523,6 +529,12 @@ void stw_unload_classes(bool full_gc); void stw_process_weak_roots(bool full_gc); + // Prepare concurrent root processing + void prepare_concurrent_roots(); + // Prepare and finish concurrent unloading + void prepare_concurrent_unloading(); + void finish_concurrent_unloading(); + // ---------- Generic interface hooks // Minor things that super-interface expects us to implement to play nice with // the rest of runtime. Some of the things here are not required to be implemented, @@ -562,7 +574,7 @@ public: void register_nmethod(nmethod* nm); void unregister_nmethod(nmethod* nm); - void flush_nmethod(nmethod* nm) {} + void flush_nmethod(nmethod* nm); void verify_nmethod(nmethod* nm) {} // ---------- Pinning hooks diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp Wed Nov 27 06:36:41 2019 -0800 +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp Wed Nov 27 11:52:57 2019 -0500 @@ -372,6 +372,14 @@ return _gc_state.is_set(UPDATEREFS); } +inline bool ShenandoahHeap::is_stw_gc_in_progress() const { + return is_full_gc_in_progress() || is_degenerated_gc_in_progress(); +} + +inline bool ShenandoahHeap::is_concurrent_root_in_progress() const { + return _concurrent_root_in_progress.is_set(); +} + template inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region, T* cl) { marked_object_iterate(region, cl, region->top()); diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahLock.cpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/hotspot/share/gc/shenandoah/shenandoahLock.cpp Wed Nov 27 11:52:57 2019 -0500 @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2019, Red Hat, Inc. All rights reserved. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "runtime/os.hpp" + +#include "gc/shenandoah/shenandoahLock.hpp" +#include "runtime/atomic.hpp" +#include "runtime/os.inline.hpp" +#include "runtime/thread.hpp" + +ShenandoahSimpleLock::ShenandoahSimpleLock() { + assert(os::mutex_init_done(), "Too early!"); +} + +void ShenandoahSimpleLock::lock() { + _lock.lock(); +} + +void ShenandoahSimpleLock::unlock() { + _lock.unlock(); +} + +ShenandoahReentrantLock::ShenandoahReentrantLock() : + ShenandoahSimpleLock(), _owner(NULL), _count(0) { + assert(os::mutex_init_done(), "Too early!"); +} + +ShenandoahReentrantLock::~ShenandoahReentrantLock() { + assert(_count == 0, "Unbalance"); +} + +void ShenandoahReentrantLock::lock() { + Thread* const thread = Thread::current(); + Thread* const owner = Atomic::load(&_owner); + + if (owner != thread) { + ShenandoahSimpleLock::lock(); + Atomic::store(&_owner, thread); + } + + _count++; +} + +void ShenandoahReentrantLock::unlock() { + assert(owned_by_self(), "Invalid owner"); + assert(_count > 0, "Invalid count"); + + _count--; + + if (_count == 0) { + Atomic::store(&_owner, (Thread*)NULL); + ShenandoahSimpleLock::unlock(); + } +} + +bool ShenandoahReentrantLock::owned_by_self() const { + Thread* const thread = Thread::current(); + Thread* const owner = Atomic::load(&_owner); + return owner == thread; +} diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahLock.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahLock.hpp Wed Nov 27 06:36:41 2019 -0800 +++ b/src/hotspot/share/gc/shenandoah/shenandoahLock.hpp Wed Nov 27 11:52:57 2019 -0500 @@ -25,6 +25,7 @@ #define SHARE_GC_SHENANDOAH_SHENANDOAHLOCK_HPP #include "memory/allocation.hpp" +#include "runtime/safepoint.hpp" #include "runtime/thread.hpp" class ShenandoahLock { @@ -96,4 +97,50 @@ } }; +class ShenandoahSimpleLock { +private: + os::PlatformMonitor _lock; // native lock +public: + ShenandoahSimpleLock(); + + virtual void lock(); + virtual void unlock(); +}; + +class ShenandoahReentrantLock : public ShenandoahSimpleLock { +private: + Thread* volatile _owner; + uint64_t _count; + +public: + ShenandoahReentrantLock(); + ~ShenandoahReentrantLock(); + + virtual void lock(); + virtual void unlock(); + + // If the lock already owned by this thread + bool owned_by_self() const ; +}; + +class ShenandoahReentrantLocker : public StackObj { +private: + ShenandoahReentrantLock* const _lock; + +public: + ShenandoahReentrantLocker(ShenandoahReentrantLock* lock) : + _lock(lock) { + if (_lock != NULL) { + _lock->lock(); + } + } + + ~ShenandoahReentrantLocker() { + if (_lock != NULL) { + assert(_lock->owned_by_self(), "Must be owner"); + _lock->unlock(); + } + } +}; + #endif // SHARE_GC_SHENANDOAH_SHENANDOAHLOCK_HPP diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp Wed Nov 27 11:52:57 2019 -0500 @@ -0,0 +1,516 @@ +/* + * Copyright (c) 2019, Red Hat, Inc. All rights reserved. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "gc/shenandoah/shenandoahClosures.inline.hpp" +#include "gc/shenandoah/shenandoahConcurrentRoots.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahNMethod.inline.hpp" +#include "memory/resourceArea.hpp" + +ShenandoahNMethod::ShenandoahNMethod(nmethod* nm, GrowableArray& oops, bool non_immediate_oops) : + _nm(nm), _oops(NULL), _oops_count(0), _unregistered(false) { + + if (!oops.is_empty()) { + _oops_count = oops.length(); + _oops = NEW_C_HEAP_ARRAY(oop*, _oops_count, mtGC); + for (int c = 0; c < _oops_count; c++) { + _oops[c] = oops.at(c); + } + } + _has_non_immed_oops = non_immediate_oops; + + assert_same_oops(); +} + +ShenandoahNMethod::~ShenandoahNMethod() { + if (_oops != NULL) { + FREE_C_HEAP_ARRAY(oop*, _oops); + } +} + +class ShenandoahHasCSetOopClosure : public OopClosure { +private: + ShenandoahHeap* const _heap; + bool _has_cset_oops; + +public: + ShenandoahHasCSetOopClosure() : + _heap(ShenandoahHeap::heap()), + _has_cset_oops(false) { + } + + bool has_cset_oops() const { + return _has_cset_oops; + } + + void do_oop(oop* p) { + oop value = RawAccess<>::oop_load(p); + if (!_has_cset_oops && _heap->in_collection_set(value)) { + _has_cset_oops = true; + } + } + + void do_oop(narrowOop* p) { + ShouldNotReachHere(); + } +}; + +bool ShenandoahNMethod::has_cset_oops(ShenandoahHeap *heap) { + ShenandoahHasCSetOopClosure cl; + oops_do(&cl); + return cl.has_cset_oops(); +} + +void ShenandoahNMethod::update() { + ResourceMark rm; + bool non_immediate_oops = false; + GrowableArray oops; + + detect_reloc_oops(nm(), oops, non_immediate_oops); + if (oops.length() != _oops_count) { + if (_oops != NULL) { + FREE_C_HEAP_ARRAY(oop*, _oops); + _oops = NULL; + } + + _oops_count = oops.length(); + if (_oops_count > 0) { + _oops = NEW_C_HEAP_ARRAY(oop*, _oops_count, mtGC); + } + } + + for (int index = 0; index < _oops_count; index ++) { + _oops[index] = oops.at(index); + } + _has_non_immed_oops = non_immediate_oops; + + assert_same_oops(); +} + +void ShenandoahNMethod::oops_do(OopClosure* oops, bool fix_relocations) { + for (int c = 0; c < _oops_count; c ++) { + oops->do_oop(_oops[c]); + } + + oop* const begin = _nm->oops_begin(); + oop* const end = _nm->oops_end(); + for (oop* p = begin; p < end; p++) { + if (*p != Universe::non_oop_word()) { + oops->do_oop(p); + } + } + + if (fix_relocations && _has_non_immed_oops) { + _nm->fix_oop_relocations(); + } +} + +void ShenandoahNMethod::detect_reloc_oops(nmethod* nm, GrowableArray& oops, bool& has_non_immed_oops) { + has_non_immed_oops = false; + // Find all oops relocations + RelocIterator iter(nm); + while (iter.next()) { + if (iter.type() != relocInfo::oop_type) { + // Not an oop + continue; + } + + oop_Relocation* r = iter.oop_reloc(); + if (!r->oop_is_immediate()) { + // Non-immediate oop found + has_non_immed_oops = true; + continue; + } + + if (r->oop_value() != NULL) { + // Non-NULL immediate oop found. NULL oops can safely be + // ignored since the method will be re-registered if they + // are later patched to be non-NULL. + oops.push(r->oop_addr()); + } + } +} + +ShenandoahNMethod* ShenandoahNMethod::for_nmethod(nmethod* nm) { + ResourceMark rm; + bool non_immediate_oops = false; + GrowableArray oops; + + detect_reloc_oops(nm, oops, non_immediate_oops); + + // No embedded oops + if(!ShenandoahConcurrentRoots::can_do_concurrent_class_unloading() && + oops.is_empty() && nm->oops_begin() >= nm->oops_end()) { + return NULL; + } + + return new ShenandoahNMethod(nm, oops, non_immediate_oops); +} + +void ShenandoahNMethod::heal_nmethod(nmethod* nm) { + ShenandoahNMethod* data = gc_data(nm); + assert(data != NULL, "Sanity"); + assert(data->lock()->owned_by_self(), "Must hold the lock"); + + ShenandoahEvacuateUpdateRootsClosure cl; + data->oops_do(&cl, true /*fix relocation*/); +} + +#ifdef ASSERT +void ShenandoahNMethod::assert_alive_and_correct() { + assert(_nm->is_alive(), "only alive nmethods here"); + ShenandoahHeap* heap = ShenandoahHeap::heap(); + for (int c = 0; c < _oops_count; c++) { + oop *loc = _oops[c]; + assert(_nm->code_contains((address) loc) || _nm->oops_contains(loc), "nmethod should contain the oop*"); + oop o = RawAccess<>::oop_load(loc); + shenandoah_assert_correct_except(loc, o, o == NULL || heap->is_full_gc_move_in_progress()); + } + + oop* const begin = _nm->oops_begin(); + oop* const end = _nm->oops_end(); + for (oop* p = begin; p < end; p++) { + if (*p != Universe::non_oop_word()) { + oop o = RawAccess<>::oop_load(p); + shenandoah_assert_correct_except(p, o, o == NULL || heap->is_full_gc_move_in_progress()); + } + } +} + +class ShenandoahNMethodOopDetector : public OopClosure { +private: + ResourceMark rm; // For growable array allocation below. + GrowableArray _oops; + +public: + ShenandoahNMethodOopDetector() : _oops(10) {}; + + void do_oop(oop* o) { + _oops.append(o); + } + void do_oop(narrowOop* o) { + fatal("NMethods should not have compressed oops embedded."); + } + + GrowableArray* oops() { + return &_oops; + } + + bool has_oops() { + return !_oops.is_empty(); + } +}; + +void ShenandoahNMethod::assert_same_oops(bool allow_dead) { + ShenandoahNMethodOopDetector detector; + nm()->oops_do(&detector, allow_dead); + + GrowableArray* oops = detector.oops(); + + assert(oops->length() == oop_count(), "Must match"); + + for (int index = 0; index < _oops_count; index ++) { + assert(oops->contains(_oops[index]), "Must contain this oop"); + } + + for (oop* p = nm()->oops_begin(); p < nm()->oops_end(); p ++) { + assert(oops->contains(p), "Must contain this oop"); + } +} + +void ShenandoahNMethod::assert_no_oops(nmethod* nm, bool allow_dead) { + ShenandoahNMethodOopDetector detector; + nm->oops_do(&detector, allow_dead); + assert(detector.oops()->length() == 0, "Should not have oops"); +} +#endif + +ShenandoahNMethodTable::ShenandoahNMethodTable() : + _heap(ShenandoahHeap::heap()), + _size(minSize), + _index(0), + _iteration_in_progress(false) { + _array = NEW_C_HEAP_ARRAY(ShenandoahNMethod*, _size, mtGC); +} + +ShenandoahNMethodTable::~ShenandoahNMethodTable() { + assert(_array != NULL, "Sanity"); + FREE_C_HEAP_ARRAY(ShenandoahNMethod*, _array); +} + +void ShenandoahNMethodTable::register_nmethod(nmethod* nm) { + assert(CodeCache_lock->owned_by_self(), "Must have CodeCache_lock held"); + assert(_index >= 0 && _index <= _size, "Sanity"); + + ShenandoahNMethod* data = ShenandoahNMethod::gc_data(nm); + ShenandoahReentrantLocker data_locker(data != NULL ? data->lock() : NULL); + + if (data != NULL) { + assert(contain(nm), "Must have been registered"); + data->update(); + } else { + data = ShenandoahNMethod::for_nmethod(nm); + if (data == NULL) { + assert(!ShenandoahConcurrentRoots::can_do_concurrent_class_unloading(), + "Only possible when concurrent class unloading is off"); + return; + } + ShenandoahNMethod::attach_gc_data(nm, data); + ShenandoahLocker locker(&_lock); + log_register_nmethod(nm); + append(data); + } + // Disarm new nmethod + ShenandoahNMethod::disarm_nmethod(nm); +} + +void ShenandoahNMethodTable::unregister_nmethod(nmethod* nm) { + assert_locked_or_safepoint(CodeCache_lock); + + ShenandoahNMethod* data = ShenandoahNMethod::gc_data(nm); + if (data == NULL) { + assert(!ShenandoahConcurrentRoots::can_do_concurrent_class_unloading(), + "Only possible when concurrent class unloading is off"); + ShenandoahNMethod::assert_no_oops(nm, true /*allow_dead*/); + return; + } + + if (Thread::current()->is_Code_cache_sweeper_thread()) { + wait_until_concurrent_iteration_done(); + } + log_unregister_nmethod(nm); + ShenandoahLocker locker(&_lock); + assert(contain(nm), "Must have been registered"); + + ShenandoahReentrantLocker data_locker(data->lock()); + data->mark_unregistered(); +} + +void ShenandoahNMethodTable::flush_nmethod(nmethod* nm) { + assert(CodeCache_lock->owned_by_self(), "Must have CodeCache_lock held"); + assert(Thread::current()->is_Code_cache_sweeper_thread(), "Must from Sweep thread"); + ShenandoahNMethod* data = ShenandoahNMethod::gc_data(nm); + assert(data != NULL || !ShenandoahConcurrentRoots::can_do_concurrent_class_unloading(), + "Only possible when concurrent class unloading is off"); + if (data == NULL) { + ShenandoahNMethod::assert_no_oops(nm, true /*allow_dead*/); + return; + } + + // Can not alter the array when iteration is in progress + wait_until_concurrent_iteration_done(); + log_flush_nmethod(nm); + + ShenandoahLocker locker(&_lock); + int idx = index_of(nm); + assert(idx >= 0 && idx < _index, "Invalid index"); + ShenandoahNMethod::attach_gc_data(nm, NULL); + remove(idx); +} + +bool ShenandoahNMethodTable::contain(nmethod* nm) const { + return index_of(nm) != -1; +} + +ShenandoahNMethod* ShenandoahNMethodTable::at(int index) const { + assert(index >= 0 && index < _index, "Out of bound"); + return _array[index]; +} + +int ShenandoahNMethodTable::index_of(nmethod* nm) const { + for (int index = 0; index < length(); index ++) { + if (_array[index]->nm() == nm) { + return index; + } + } + return -1; +} + +void ShenandoahNMethodTable::remove(int idx) { + shenandoah_assert_locked_or_safepoint(CodeCache_lock); + assert(!_iteration_in_progress, "Can not happen"); + assert(_index >= 0 && _index <= _size, "Sanity"); + + assert(idx >= 0 && idx < _index, "Out of bound"); + ShenandoahNMethod* snm = _array[idx]; + + _index --; + _array[idx] = _array[_index]; + + delete snm; +} + +void ShenandoahNMethodTable::wait_until_concurrent_iteration_done() { + assert(CodeCache_lock->owned_by_self(), "Lock must be held"); + while (iteration_in_progress()) { + CodeCache_lock->wait_without_safepoint_check(); + } +} + +void ShenandoahNMethodTable::append(ShenandoahNMethod* snm) { + if (is_full()) { + int new_size = 2 * _size; + ShenandoahNMethod** old_table = _array; + + // Rebuild table and replace current one + rebuild(new_size); + + // An iteration is in progress over early snapshot, + // can not release the array until iteration is completed + if (!iteration_in_progress()) { + FREE_C_HEAP_ARRAY(ShenandoahNMethod*, old_table); + } + } + + _array[_index ++] = snm; + assert(_index >= 0 && _index <= _size, "Sanity"); +} + +void ShenandoahNMethodTable::rebuild(int size) { + ShenandoahNMethod** arr = NEW_C_HEAP_ARRAY(ShenandoahNMethod*, size, mtGC); + for (int index = 0; index < _index; index ++) { + arr[index] = _array[index]; + } + _array = arr; + _size = size; +} + +ShenandoahNMethodTableSnapshot* ShenandoahNMethodTable::snapshot_for_iteration() { + assert(!iteration_in_progress(), "Already in progress"); + _iteration_in_progress = true; + + return new ShenandoahNMethodTableSnapshot(this); +} + +void ShenandoahNMethodTable::finish_iteration(ShenandoahNMethodTableSnapshot* snapshot) { + assert(iteration_in_progress(), "Why we here?"); + assert(snapshot != NULL, "No snapshot"); + _iteration_in_progress = false; + + // Table has been rebuilt during iteration, free old table + if (snapshot->_array != _array) { + FREE_C_HEAP_ARRAY(ShenandoahNMethod*, snapshot->_array); + } + delete snapshot; +} + +void ShenandoahNMethodTable::log_register_nmethod(nmethod* nm) { + LogTarget(Debug, gc, nmethod) log; + if (!log.is_enabled()) { + return; + } + + ResourceMark rm; + log.print("Register NMethod: %s.%s [" PTR_FORMAT "] (%s)", + nm->method()->method_holder()->external_name(), + nm->method()->name()->as_C_string(), + p2i(nm), + nm->compiler_name()); +} + +void ShenandoahNMethodTable::log_unregister_nmethod(nmethod* nm) { + LogTarget(Debug, gc, nmethod) log; + if (!log.is_enabled()) { + return; + } + + ResourceMark rm; + log.print("Unregister NMethod: %s.%s [" PTR_FORMAT "]", + nm->method()->method_holder()->external_name(), + nm->method()->name()->as_C_string(), + p2i(nm)); +} + +void ShenandoahNMethodTable::log_flush_nmethod(nmethod* nm) { + LogTarget(Debug, gc, nmethod) log; + if (!log.is_enabled()) { + return; + } + + ResourceMark rm; + log.print("Flush NMethod: (" PTR_FORMAT ")", p2i(nm)); +} + +#ifdef ASSERT +void ShenandoahNMethodTable::assert_nmethods_alive_and_correct() { + assert_locked_or_safepoint(CodeCache_lock); + + for (int index = 0; index < length(); index ++) { + ShenandoahNMethod* m = _array[index]; + // Concurrent unloading may have dead nmethods to be cleaned by sweeper + if (m->is_unregistered()) continue; + m->assert_alive_and_correct(); + } +} +#endif + +ShenandoahNMethodTableSnapshot::ShenandoahNMethodTableSnapshot(ShenandoahNMethodTable* table) : + _heap(ShenandoahHeap::heap()), _table(table), _array(table->_array), _length(table->_index), _claimed(0) { +} + +void ShenandoahNMethodTableSnapshot::concurrent_nmethods_do(NMethodClosure* cl) { + size_t stride = 256; // educated guess + + ShenandoahNMethod** list = _array; + size_t max = (size_t)_length; + while (_claimed < max) { + size_t cur = Atomic::add(&_claimed, stride) - stride; + size_t start = cur; + size_t end = MIN2(cur + stride, max); + if (start >= max) break; + + for (size_t idx = start; idx < end; idx++) { + ShenandoahNMethod* data = list[idx]; + assert(data != NULL, "Should not be NULL"); + if (!data->is_unregistered()) { + cl->do_nmethod(data->nm()); + } + } + } +} + +ShenandoahConcurrentNMethodIterator::ShenandoahConcurrentNMethodIterator(ShenandoahNMethodTable* table) : + _table(table), _table_snapshot(NULL) { +} + +void ShenandoahConcurrentNMethodIterator::nmethods_do_begin() { + assert(CodeCache_lock->owned_by_self(), "Lock must be held"); + assert(ShenandoahConcurrentRoots::can_do_concurrent_class_unloading(), + "Only for concurrent class unloading"); + _table_snapshot = _table->snapshot_for_iteration(); +} + +void ShenandoahConcurrentNMethodIterator::nmethods_do(NMethodClosure* cl) { + assert(_table_snapshot != NULL, "Must first call nmethod_do_begin()"); + _table_snapshot->concurrent_nmethods_do(cl); +} + +void ShenandoahConcurrentNMethodIterator::nmethods_do_end() { + assert(CodeCache_lock->owned_by_self(), "Lock must be held"); + assert(ShenandoahConcurrentRoots::can_do_concurrent_class_unloading(), + "Only for concurrent class unloading"); + _table->finish_iteration(_table_snapshot); + CodeCache_lock->notify_all(); +} diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahNMethod.hpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/hotspot/share/gc/shenandoah/shenandoahNMethod.hpp Wed Nov 27 11:52:57 2019 -0500 @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2019, Red Hat, Inc. All rights reserved. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHNMETHOD_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHNMETHOD_HPP + +#include "code/nmethod.hpp" +#include "gc/shenandoah/shenandoahHeap.hpp" +#include "gc/shenandoah/shenandoahLock.hpp" +#include "memory/allocation.hpp" +#include "utilities/growableArray.hpp" + +// ShenandoahNMethod tuple records the internal locations of oop slots within reclocation stream in +// the nmethod. This allows us to quickly scan the oops without doing the nmethod-internal scans, +// that sometimes involves parsing the machine code. Note it does not record the oops themselves, +// because it would then require handling these tuples as the new class of roots. +class ShenandoahNMethod : public CHeapObj { +private: + nmethod* const _nm; + oop** _oops; + int _oops_count; + bool _has_non_immed_oops; + bool _unregistered; + ShenandoahReentrantLock _lock; + +public: + ShenandoahNMethod(nmethod *nm, GrowableArray& oops, bool has_non_immed_oops); + ~ShenandoahNMethod(); + + inline nmethod* nm() const; + inline ShenandoahReentrantLock* lock(); + void oops_do(OopClosure* oops, bool fix_relocations = false); + // Update oops when the nmethod is re-registered + void update(); + + bool has_cset_oops(ShenandoahHeap* heap); + + inline int oop_count() const; + inline bool has_oops() const; + + inline void mark_unregistered(); + inline bool is_unregistered() const; + + static ShenandoahNMethod* for_nmethod(nmethod* nm); + static inline ShenandoahReentrantLock* lock_for_nmethod(nmethod* nm); + + static void heal_nmethod(nmethod* nm); + static inline void disarm_nmethod(nmethod* nm); + + static inline ShenandoahNMethod* gc_data(nmethod* nm); + static inline void attach_gc_data(nmethod* nm, ShenandoahNMethod* gc_data); + + void assert_alive_and_correct() NOT_DEBUG_RETURN; + void assert_same_oops(bool allow_dead = false) NOT_DEBUG_RETURN; + static void assert_no_oops(nmethod* nm, bool allow_dea = false) NOT_DEBUG_RETURN; + +private: + bool has_non_immed_oops() const { return _has_non_immed_oops; } + static void detect_reloc_oops(nmethod* nm, GrowableArray& oops, bool& _has_non_immed_oops); +}; + +class ShenandoahNMethodTable; + +// An opaque snapshot of current nmethod table for iteration +class ShenandoahNMethodTableSnapshot : public CHeapObj { + friend class ShenandoahNMethodTable; +private: + ShenandoahHeap* const _heap; + ShenandoahNMethodTable* _table; + ShenandoahNMethod** const _array; + const int _length; + + DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile size_t)); + volatile size_t _claimed; + DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0); + +public: + ShenandoahNMethodTableSnapshot(ShenandoahNMethodTable* table); + + template + void parallel_blobs_do(CodeBlobClosure *f); + + void concurrent_nmethods_do(NMethodClosure* cl); +}; + +class ShenandoahNMethodTable : public CHeapObj { + friend class ShenandoahNMethodTableSnapshot; +private: + enum { + minSize = 1024 + }; + + ShenandoahHeap* const _heap; + ShenandoahNMethod** _array; + int _size; + int _index; + ShenandoahLock _lock; + bool _iteration_in_progress; + +public: + ShenandoahNMethodTable(); + ~ShenandoahNMethodTable(); + + void register_nmethod(nmethod* nm); + void unregister_nmethod(nmethod* nm); + void flush_nmethod(nmethod* nm); + + bool contain(nmethod* nm) const; + int length() const { return _index; } + + // Table iteration support + ShenandoahNMethodTableSnapshot* snapshot_for_iteration(); + void finish_iteration(ShenandoahNMethodTableSnapshot* snapshot); + + void assert_nmethods_alive_and_correct() NOT_DEBUG_RETURN; +private: + // Rebuild table and replace current one + void rebuild(int size); + + bool is_full() const { + assert(_index <= _size, "Sanity"); + return _index == _size; + } + + ShenandoahNMethod* at(int index) const; + int index_of(nmethod* nm) const; + void remove(int index); + void append(ShenandoahNMethod* snm); + + inline bool iteration_in_progress() const; + void wait_until_concurrent_iteration_done(); + + // Logging support + void log_register_nmethod(nmethod* nm); + void log_unregister_nmethod(nmethod* nm); + void log_flush_nmethod(nmethod* nm); +}; + +class ShenandoahConcurrentNMethodIterator { +private: + ShenandoahNMethodTable* const _table; + ShenandoahNMethodTableSnapshot* _table_snapshot; + +public: + ShenandoahConcurrentNMethodIterator(ShenandoahNMethodTable* table); + + void nmethods_do_begin(); + void nmethods_do(NMethodClosure* cl); + void nmethods_do_end(); +}; + +#endif // SHARE_GC_SHENANDOAH_SHENANDOAHNMETHOD_HPP diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp Wed Nov 27 11:52:57 2019 -0500 @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2019, Red Hat, Inc. All rights reserved. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHNMETHOD_INLINE_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHNMETHOD_INLINE_HPP + +#include "gc/shared/barrierSet.hpp" +#include "gc/shared/barrierSetNMethod.hpp" +#include "gc/shenandoah/shenandoahConcurrentRoots.hpp" +#include "gc/shenandoah/shenandoahNMethod.hpp" + +nmethod* ShenandoahNMethod::nm() const { + return _nm; +} + +ShenandoahReentrantLock* ShenandoahNMethod::lock() { + return &_lock; +} + +int ShenandoahNMethod::oop_count() const { + return _oops_count + static_cast(nm()->oops_end() - nm()->oops_begin()); +} + +bool ShenandoahNMethod::has_oops() const { + return oop_count() > 0; +} + +void ShenandoahNMethod::mark_unregistered() { + _unregistered = true; +} + +bool ShenandoahNMethod::is_unregistered() const { + return _unregistered; +} + +void ShenandoahNMethod::disarm_nmethod(nmethod* nm) { + if (!ShenandoahConcurrentRoots::can_do_concurrent_class_unloading()) { + return; + } + + BarrierSetNMethod* const bs = BarrierSet::barrier_set()->barrier_set_nmethod(); + assert(bs != NULL, "Sanity"); + bs->disarm(nm); +} + +ShenandoahNMethod* ShenandoahNMethod::gc_data(nmethod* nm) { + return nm->gc_data(); +} + +void ShenandoahNMethod::attach_gc_data(nmethod* nm, ShenandoahNMethod* gc_data) { + nm->set_gc_data(gc_data); +} + +ShenandoahReentrantLock* ShenandoahNMethod::lock_for_nmethod(nmethod* nm) { + return gc_data(nm)->lock(); +} + +bool ShenandoahNMethodTable::iteration_in_progress() const { + return _iteration_in_progress; +} + +template +void ShenandoahNMethodTableSnapshot::parallel_blobs_do(CodeBlobClosure *f) { + size_t stride = 256; // educated guess + + ShenandoahNMethod** const list = _array; + + size_t max = (size_t)_length; + while (_claimed < max) { + size_t cur = Atomic::add(&_claimed, stride) - stride; + size_t start = cur; + size_t end = MIN2(cur + stride, max); + if (start >= max) break; + + for (size_t idx = start; idx < end; idx++) { + ShenandoahNMethod* nmr = list[idx]; + assert(nmr != NULL, "Sanity"); + if (nmr->is_unregistered()) { + continue; + } + + nmr->assert_alive_and_correct(); + + if (CSET_FILTER && !nmr->has_cset_oops(_heap)) { + continue; + } + + f->do_code_blob(nmr->nm()); + } + } +} + +#endif // SHARE_GC_SHENANDOAH_SHENANDOAHNMETHOD_INLINE_HPP diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahNormalMode.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahNormalMode.cpp Wed Nov 27 06:36:41 2019 -0800 +++ b/src/hotspot/share/gc/shenandoah/shenandoahNormalMode.cpp Wed Nov 27 11:52:57 2019 -0500 @@ -21,6 +21,7 @@ */ #include "precompiled.hpp" +#include "gc/shenandoah/shenandoahConcurrentRoots.hpp" #include "gc/shenandoah/shenandoahNormalMode.hpp" #include "gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp" #include "gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp" @@ -32,6 +33,9 @@ void ShenandoahNormalMode::initialize_flags() const { SHENANDOAH_ERGO_ENABLE_FLAG(ExplicitGCInvokesConcurrent); SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahImplicitGCInvokesConcurrent); + if (ShenandoahConcurrentRoots::can_do_concurrent_class_unloading()) { + SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahSuspendibleWorkers); + } // Final configuration checks SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier); diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp Wed Nov 27 06:36:41 2019 -0800 +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp Wed Nov 27 11:52:57 2019 -0500 @@ -27,8 +27,10 @@ #include "classfile/stringTable.hpp" #include "classfile/systemDictionary.hpp" #include "code/codeCache.hpp" +#include "gc/shenandoah/shenandoahClosures.inline.hpp" +#include "gc/shenandoah/shenandoahConcurrentRoots.hpp" #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp" -#include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahHeap.hpp" #include "gc/shenandoah/shenandoahPhaseTimings.hpp" #include "gc/shenandoah/shenandoahStringDedup.hpp" #include "gc/shenandoah/shenandoahTimingTracker.hpp" @@ -159,14 +161,22 @@ _heap->phase_timings()->record_workers_end(_phase); } -ShenandoahRootEvacuator::ShenandoahRootEvacuator(uint n_workers, ShenandoahPhaseTimings::Phase phase, bool include_concurrent_roots) : +ShenandoahRootEvacuator::ShenandoahRootEvacuator(uint n_workers, + ShenandoahPhaseTimings::Phase phase, + bool include_concurrent_roots, + bool include_concurrent_code_roots) : ShenandoahRootProcessor(phase), _thread_roots(n_workers > 1), - _include_concurrent_roots(include_concurrent_roots) { + _include_concurrent_roots(include_concurrent_roots), + _include_concurrent_code_roots(include_concurrent_code_roots) { } void ShenandoahRootEvacuator::roots_do(uint worker_id, OopClosure* oops) { MarkingCodeBlobClosure blobsCl(oops, CodeBlobToOopClosure::FixRelocations); + ShenandoahCodeBlobAndDisarmClosure blobs_and_disarm_Cl(oops); + CodeBlobToOopClosure* codes_cl = ShenandoahConcurrentRoots::can_do_concurrent_class_unloading() ? + static_cast(&blobs_and_disarm_Cl) : + static_cast(&blobsCl); AlwaysTrueClosure always_true; _serial_roots.oops_do(oops, worker_id); @@ -178,8 +188,12 @@ _weak_roots.oops_do(oops, worker_id); } - _thread_roots.oops_do(oops, NULL, worker_id); - _code_roots.code_blobs_do(&blobsCl, worker_id); + if (_include_concurrent_code_roots) { + _code_roots.code_blobs_do(codes_cl, worker_id); + _thread_roots.oops_do(oops, NULL, worker_id); + } else { + _thread_roots.oops_do(oops, codes_cl, worker_id); + } _dedup_roots.oops_do(&always_true, oops, worker_id); } @@ -208,7 +222,11 @@ } void ShenandoahRootAdjuster::roots_do(uint worker_id, OopClosure* oops) { - CodeBlobToOopClosure adjust_code_closure(oops, CodeBlobToOopClosure::FixRelocations); + CodeBlobToOopClosure code_blob_cl(oops, CodeBlobToOopClosure::FixRelocations); + ShenandoahCodeBlobAndDisarmClosure blobs_and_disarm_Cl(oops); + CodeBlobToOopClosure* adjust_code_closure = ShenandoahConcurrentRoots::can_do_concurrent_class_unloading() ? + static_cast(&blobs_and_disarm_Cl) : + static_cast(&code_blob_cl); CLDToOopClosure adjust_cld_closure(oops, ClassLoaderData::_claim_strong); AlwaysTrueClosure always_true; @@ -217,7 +235,7 @@ _thread_roots.oops_do(oops, NULL, worker_id); _cld_roots.cld_do(&adjust_cld_closure, worker_id); - _code_roots.code_blobs_do(&adjust_code_closure, worker_id); + _code_roots.code_blobs_do(adjust_code_closure, worker_id); _serial_weak_roots.weak_oops_do(oops, worker_id); _weak_roots.oops_do(oops, worker_id); diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp Wed Nov 27 06:36:41 2019 -0800 +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp Wed Nov 27 11:52:57 2019 -0500 @@ -277,11 +277,12 @@ ShenandoahSerialWeakRoots _serial_weak_roots; ShenandoahWeakRoots _weak_roots; ShenandoahStringDedupRoots _dedup_roots; - ShenandoahCodeCacheRoots _code_roots; + ShenandoahCodeCacheRoots _code_roots; bool _include_concurrent_roots; - + bool _include_concurrent_code_roots; public: - ShenandoahRootEvacuator(uint n_workers, ShenandoahPhaseTimings::Phase phase, bool include_concurrent_roots); + ShenandoahRootEvacuator(uint n_workers, ShenandoahPhaseTimings::Phase phase, + bool include_concurrent_roots, bool _include_concurrent_code_roots); void roots_do(uint worker_id, OopClosure* oops); }; @@ -297,7 +298,7 @@ ShenandoahSerialWeakRoots _serial_weak_roots; ShenandoahWeakRoots _weak_roots; ShenandoahStringDedupRoots _dedup_roots; - ShenandoahCodeCacheRoots _code_roots; + ShenandoahCodeCacheRoots _code_roots; public: ShenandoahRootUpdater(uint n_workers, ShenandoahPhaseTimings::Phase phase); diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.inline.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.inline.hpp Wed Nov 27 06:36:41 2019 -0800 +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.inline.hpp Wed Nov 27 11:52:57 2019 -0500 @@ -28,6 +28,8 @@ #include "classfile/stringTable.hpp" #include "classfile/systemDictionary.hpp" #include "gc/shared/oopStorageParState.inline.hpp" +#include "gc/shenandoah/shenandoahClosures.inline.hpp" +#include "gc/shenandoah/shenandoahConcurrentRoots.hpp" #include "gc/shenandoah/shenandoahHeuristics.hpp" #include "gc/shenandoah/shenandoahRootProcessor.hpp" #include "gc/shenandoah/shenandoahTimingTracker.hpp" @@ -265,14 +267,19 @@ template void ShenandoahRootUpdater::roots_do(uint worker_id, IsAlive* is_alive, KeepAlive* keep_alive) { CodeBlobToOopClosure update_blobs(keep_alive, CodeBlobToOopClosure::FixRelocations); + ShenandoahCodeBlobAndDisarmClosure blobs_and_disarm_Cl(keep_alive); + CodeBlobToOopClosure* codes_cl = ShenandoahConcurrentRoots::can_do_concurrent_class_unloading() ? + static_cast(&blobs_and_disarm_Cl) : + static_cast(&update_blobs); + CLDToOopClosure clds(keep_alive, ClassLoaderData::_claim_strong); _serial_roots.oops_do(keep_alive, worker_id); _vm_roots.oops_do(keep_alive, worker_id); - _thread_roots.oops_do(keep_alive, NULL, worker_id); _cld_roots.cld_do(&clds, worker_id); _code_roots.code_blobs_do(&update_blobs, worker_id); + _thread_roots.oops_do(keep_alive, NULL, worker_id); _serial_weak_roots.weak_oops_do(is_alive, keep_alive, worker_id); _weak_roots.weak_oops_do(is_alive, keep_alive, worker_id); diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp Wed Nov 27 06:36:41 2019 -0800 +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp Wed Nov 27 11:52:57 2019 -0500 @@ -43,7 +43,7 @@ // Check for overflow of number of root types. STATIC_ASSERT((static_cast(ShenandoahRootVerifier::AllRoots) + 1) > static_cast(ShenandoahRootVerifier::AllRoots)); -ShenandoahRootVerifier::ShenandoahRootVerifier() : _types(AllRoots) { +ShenandoahRootVerifier::ShenandoahRootVerifier(RootTypes types) : _types(types) { } void ShenandoahRootVerifier::excludes(RootTypes types) { diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp Wed Nov 27 06:36:41 2019 -0800 +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp Wed Nov 27 11:52:57 2019 -0500 @@ -30,6 +30,7 @@ class ShenandoahRootVerifier : public StackObj { public: enum RootTypes { + None = 0, SerialRoots = 1 << 0, ThreadRoots = 1 << 1, CodeRoots = 1 << 2, @@ -44,7 +45,7 @@ RootTypes _types; public: - ShenandoahRootVerifier(); + ShenandoahRootVerifier(RootTypes types = AllRoots); void excludes(RootTypes types); void oops_do(OopClosure* cl); diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp Wed Nov 27 06:36:41 2019 -0800 +++ b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp Wed Nov 27 11:52:57 2019 -0500 @@ -26,6 +26,7 @@ #include "gc/shared/plab.hpp" #include "gc/shenandoah/shenandoahBarrierSet.hpp" +#include "gc/shenandoah/shenandoahCodeRoots.hpp" #include "gc/shenandoah/shenandoahSATBMarkQueueSet.hpp" #include "runtime/thread.hpp" #include "utilities/debug.hpp" @@ -43,6 +44,7 @@ size_t _gclab_size; uint _worker_id; bool _force_satb_flush; + int _disarmed_value; ShenandoahThreadLocalData() : _gc_state(0), @@ -125,6 +127,7 @@ assert(data(thread)->_gclab == NULL, "Only initialize once"); data(thread)->_gclab = new PLAB(PLAB::min_size()); data(thread)->_gclab_size = 0; + data(thread)->_disarmed_value = ShenandoahCodeRoots::disarmed_value(); } static PLAB* gclab(Thread* thread) { @@ -139,6 +142,10 @@ data(thread)->_gclab_size = v; } + static void set_disarmed_value(Thread* thread, int value) { + data(thread)->_disarmed_value = value; + } + #ifdef ASSERT static void set_evac_allowed(Thread* thread, bool evac_allowed) { if (evac_allowed) { @@ -170,6 +177,9 @@ return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _gc_state); } + static ByteSize disarmed_value_offset() { + return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _disarmed_value); + } }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHTHREADLOCALDATA_HPP diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp Wed Nov 27 11:52:57 2019 -0500 @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2019, Red Hat, Inc. All rights reserved. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "classfile/classLoaderDataGraph.hpp" +#include "classfile/systemDictionary.hpp" +#include "code/codeBehaviours.hpp" +#include "code/codeCache.hpp" +#include "code/dependencyContext.hpp" +#include "gc/shared/gcBehaviours.hpp" +#include "gc/shared/suspendibleThreadSet.hpp" +#include "gc/shenandoah/shenandoahClosures.inline.hpp" +#include "gc/shenandoah/shenandoahCodeRoots.hpp" +#include "gc/shenandoah/shenandoahConcurrentRoots.hpp" +#include "gc/shenandoah/shenandoahNMethod.inline.hpp" +#include "gc/shenandoah/shenandoahLock.hpp" +#include "gc/shenandoah/shenandoahRootProcessor.hpp" +#include "gc/shenandoah/shenandoahUnload.hpp" +#include "gc/shenandoah/shenandoahVerifier.hpp" +#include "memory/iterator.hpp" +#include "memory/resourceArea.hpp" +#include "oops/access.inline.hpp" + +class ShenandoahIsUnloadingOopClosure : public OopClosure { +private: + ShenandoahMarkingContext* _marking_context; + bool _is_unloading; + +public: + ShenandoahIsUnloadingOopClosure() : + _marking_context(ShenandoahHeap::heap()->marking_context()), + _is_unloading(false) { + } + + virtual void do_oop(oop* p) { + if (_is_unloading) { + return; + } + + const oop o = RawAccess<>::oop_load(p); + if (!CompressedOops::is_null(o) && + _marking_context->is_complete() && + !_marking_context->is_marked(o)) { + _is_unloading = true; + } + } + + virtual void do_oop(narrowOop* p) { + ShouldNotReachHere(); + } + + bool is_unloading() const { + return _is_unloading; + } +}; + +class ShenandoahIsUnloadingBehaviour : public IsUnloadingBehaviour { +public: + virtual bool is_unloading(CompiledMethod* method) const { + nmethod* const nm = method->as_nmethod(); + guarantee(ShenandoahHeap::heap()->is_evacuation_in_progress(), "Only this phase"); + ShenandoahNMethod* data = ShenandoahNMethod::gc_data(nm); + ShenandoahReentrantLocker locker(data->lock()); + ShenandoahIsUnloadingOopClosure cl; + data->oops_do(&cl); + return cl.is_unloading(); + } +}; + +class ShenandoahCompiledICProtectionBehaviour : public CompiledICProtectionBehaviour { +public: + virtual bool lock(CompiledMethod* method) { + nmethod* const nm = method->as_nmethod(); + ShenandoahReentrantLock* const lock = ShenandoahNMethod::lock_for_nmethod(nm); + assert(lock != NULL, "Not yet registered?"); + lock->lock(); + return true; + } + + virtual void unlock(CompiledMethod* method) { + nmethod* const nm = method->as_nmethod(); + ShenandoahReentrantLock* const lock = ShenandoahNMethod::lock_for_nmethod(nm); + assert(lock != NULL, "Not yet registered?"); + lock->unlock(); + } + + virtual bool is_safe(CompiledMethod* method) { + if (SafepointSynchronize::is_at_safepoint()) { + return true; + } + + nmethod* const nm = method->as_nmethod(); + ShenandoahReentrantLock* const lock = ShenandoahNMethod::lock_for_nmethod(nm); + assert(lock != NULL, "Not yet registered?"); + return lock->owned_by_self(); + } +}; + +ShenandoahUnload::ShenandoahUnload() { + if (ShenandoahConcurrentRoots::can_do_concurrent_class_unloading()) { + static ShenandoahIsUnloadingBehaviour is_unloading_behaviour; + IsUnloadingBehaviour::set_current(&is_unloading_behaviour); + + static ShenandoahCompiledICProtectionBehaviour ic_protection_behaviour; + CompiledICProtectionBehaviour::set_current(&ic_protection_behaviour); + } +} + +void ShenandoahUnload::prepare() { + assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); + assert(ShenandoahConcurrentRoots::can_do_concurrent_class_unloading(), "Sanity"); + CodeCache::increment_unloading_cycle(); + DependencyContext::cleaning_start(); +} + +void ShenandoahUnload::unlink() { + SuspendibleThreadSetJoiner sts; + bool unloading_occurred; + ShenandoahHeap* const heap = ShenandoahHeap::heap(); + { + MutexLocker cldg_ml(ClassLoaderDataGraph_lock); + unloading_occurred = SystemDictionary::do_unloading(heap->gc_timer()); + } + + Klass::clean_weak_klass_links(unloading_occurred); + ShenandoahCodeRoots::unlink(ShenandoahHeap::heap()->workers(), unloading_occurred); + DependencyContext::cleaning_end(); +} + +void ShenandoahUnload::purge() { + { + SuspendibleThreadSetJoiner sts; + ShenandoahCodeRoots::purge(ShenandoahHeap::heap()->workers()); + } + + ClassLoaderDataGraph::purge(); + CodeCache::purge_exception_caches(); +} + +class ShenandoahUnloadRendezvousClosure : public ThreadClosure { +public: + void do_thread(Thread* thread) {} +}; + +void ShenandoahUnload::unload() { + assert(ShenandoahConcurrentRoots::can_do_concurrent_class_unloading(), "Why we here?"); + if (!ShenandoahHeap::heap()->is_evacuation_in_progress()) { + return; + } + + // Unlink stale metadata and nmethods + unlink(); + + // Make sure stale metadata and nmethods are no longer observable + ShenandoahUnloadRendezvousClosure cl; + Handshake::execute(&cl); + + // Purge stale metadata and nmethods that were unlinked + purge(); +} + +void ShenandoahUnload::finish() { + MetaspaceGC::compute_new_size(); + MetaspaceUtils::verify_metrics(); +} diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahUnload.hpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/hotspot/share/gc/shenandoah/shenandoahUnload.hpp Wed Nov 27 11:52:57 2019 -0500 @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2019, Red Hat, Inc. All rights reserved. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHCLASSUNLOAD_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHCLASSUNLOAD_HPP + +#include "memory/allocation.hpp" + +class ShenandoahHeap; + +class ShenandoahUnload { +public: + ShenandoahUnload(); + void prepare(); + void unload(); + void finish(); +private: + void unlink(); + void purge(); +}; + +#endif // SHARE_GC_SHENANDOAH_SHENANDOAHCLASSUNLOAD_HPP diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp Wed Nov 27 06:36:41 2019 -0800 +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp Wed Nov 27 11:52:57 2019 -0500 @@ -1009,6 +1009,13 @@ verifier.oops_do(&cl); } +void ShenandoahVerifier::verify_roots_in_to_space_except(ShenandoahRootVerifier::RootTypes types) { + ShenandoahRootVerifier verifier; + verifier.excludes(types); + ShenandoahVerifyInToSpaceClosure cl; + verifier.oops_do(&cl); +} + void ShenandoahVerifier::verify_roots_no_forwarded() { ShenandoahRootVerifier verifier; ShenandoahVerifyNoForwared cl; diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp Wed Nov 27 06:36:41 2019 -0800 +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp Wed Nov 27 11:52:57 2019 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2018, Red Hat, Inc. All rights reserved. + * Copyright (c) 2017, 2019, Red Hat, Inc. All rights reserved. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as @@ -190,6 +190,8 @@ // Roots should only contain to-space oops void verify_roots_in_to_space(); + void verify_roots_in_to_space_except(ShenandoahRootVerifier::RootTypes types); + void verify_roots_no_forwarded(); void verify_roots_no_forwarded_except(ShenandoahRootVerifier::RootTypes types); }; diff -r 8b6cc0bb93d0 -r 9186be5c78ba src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp Wed Nov 27 06:36:41 2019 -0800 +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp Wed Nov 27 11:52:57 2019 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2018, Red Hat, Inc. All rights reserved. + * Copyright (c) 2016, 2019, Red Hat, Inc. All rights reserved. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as @@ -86,7 +86,7 @@ "References get processed at every Nth GC cycle. Set to zero " \ "to disable reference processing.") \ \ - experimental(uintx, ShenandoahUnloadClassesFrequency, 5, \ + experimental(uintx, ShenandoahUnloadClassesFrequency, 1, \ "How often should classes get unloaded. " \ "Class unloading is performed at every Nth GC cycle. " \ "Set to zero to disable class unloading during concurrent GC.") \ diff -r 8b6cc0bb93d0 -r 9186be5c78ba test/hotspot/jtreg/gc/shenandoah/options/TestClassUnloadingArguments.java --- a/test/hotspot/jtreg/gc/shenandoah/options/TestClassUnloadingArguments.java Wed Nov 27 06:36:41 2019 -0800 +++ b/test/hotspot/jtreg/gc/shenandoah/options/TestClassUnloadingArguments.java Wed Nov 27 11:52:57 2019 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, Red Hat, Inc. All rights reserved. + * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as @@ -81,7 +81,7 @@ public static void testShenandoah() throws Exception { testWith("Shenandoah GC should have class unloading enabled", - true, false, + true, true, "-XX:+UnlockExperimentalVMOptions", "-XX:+UseShenandoahGC");