# HG changeset patch # User eosterlund # Date 1539688458 -7200 # Node ID 3a168f782e80ff2a9f7bb9726522cff07d94f5a7 # Parent 5a2af44ecb83748f455e1236f9afbf2cf6de7d0f 8210064: ZGC: Introduce ZConcurrentRootsIterator for scanning a subset of strong IN_NATIVE roots concurrently Reviewed-by: pliden, kbarrett diff -r 5a2af44ecb83 -r 3a168f782e80 src/hotspot/share/classfile/classLoaderDataGraph.cpp --- a/src/hotspot/share/classfile/classLoaderDataGraph.cpp Tue Oct 16 13:43:04 2018 +0200 +++ b/src/hotspot/share/classfile/classLoaderDataGraph.cpp Tue Oct 16 13:14:18 2018 +0200 @@ -231,14 +231,14 @@ } void ClassLoaderDataGraph::cld_do(CLDClosure* cl) { - assert_locked_or_safepoint(ClassLoaderDataGraph_lock); + assert_locked_or_safepoint_weak(ClassLoaderDataGraph_lock); for (ClassLoaderData* cld = _head; cld != NULL; cld = cld->_next) { cl->do_cld(cld); } } void ClassLoaderDataGraph::cld_unloading_do(CLDClosure* cl) { - assert_locked_or_safepoint(ClassLoaderDataGraph_lock); + assert_locked_or_safepoint_weak(ClassLoaderDataGraph_lock); // Only walk the head until any clds not purged from prior unloading // (CMS doesn't purge right away). for (ClassLoaderData* cld = _unloading; cld != _saved_unloading; cld = cld->next()) { @@ -248,7 +248,7 @@ } void ClassLoaderDataGraph::roots_cld_do(CLDClosure* strong, CLDClosure* weak) { - assert_locked_or_safepoint(ClassLoaderDataGraph_lock); + assert_locked_or_safepoint_weak(ClassLoaderDataGraph_lock); for (ClassLoaderData* cld = _head; cld != NULL; cld = cld->_next) { CLDClosure* closure = cld->keep_alive() ? strong : weak; if (closure != NULL) { @@ -258,7 +258,7 @@ } void ClassLoaderDataGraph::always_strong_cld_do(CLDClosure* cl) { - assert_locked_or_safepoint(ClassLoaderDataGraph_lock); + assert_locked_or_safepoint_weak(ClassLoaderDataGraph_lock); if (ClassUnloading) { roots_cld_do(cl, NULL); } else { diff -r 5a2af44ecb83 -r 3a168f782e80 src/hotspot/share/gc/z/zCollectedHeap.cpp --- a/src/hotspot/share/gc/z/zCollectedHeap.cpp Tue Oct 16 13:43:04 2018 +0200 +++ b/src/hotspot/share/gc/z/zCollectedHeap.cpp Tue Oct 16 13:14:18 2018 +0200 @@ -23,6 +23,7 @@ #include "precompiled.hpp" #include "gc/shared/gcHeapSummary.hpp" +#include "gc/shared/suspendibleThreadSet.hpp" #include "gc/z/zCollectedHeap.hpp" #include "gc/z/zGlobals.hpp" #include "gc/z/zHeap.inline.hpp" @@ -295,6 +296,14 @@ reserved_region().start() + max_capacity_in_words); } +void ZCollectedHeap::safepoint_synchronize_begin() { + SuspendibleThreadSet::synchronize(); +} + +void ZCollectedHeap::safepoint_synchronize_end() { + SuspendibleThreadSet::desynchronize(); +} + void ZCollectedHeap::prepare_for_verify() { // Does nothing } diff -r 5a2af44ecb83 -r 3a168f782e80 src/hotspot/share/gc/z/zCollectedHeap.hpp --- a/src/hotspot/share/gc/z/zCollectedHeap.hpp Tue Oct 16 13:43:04 2018 +0200 +++ b/src/hotspot/share/gc/z/zCollectedHeap.hpp Tue Oct 16 13:14:18 2018 +0200 @@ -117,6 +117,9 @@ virtual VirtualSpaceSummary create_heap_space_summary(); + virtual void safepoint_synchronize_begin(); + virtual void safepoint_synchronize_end(); + virtual void print_on(outputStream* st) const; virtual void print_on_error(outputStream* st) const; virtual void print_extended_on(outputStream* st) const; diff -r 5a2af44ecb83 -r 3a168f782e80 src/hotspot/share/gc/z/zDriver.cpp --- a/src/hotspot/share/gc/z/zDriver.cpp Tue Oct 16 13:43:04 2018 +0200 +++ b/src/hotspot/share/gc/z/zDriver.cpp Tue Oct 16 13:14:18 2018 +0200 @@ -336,7 +336,7 @@ // Phase 2: Concurrent Mark { ZStatTimer timer(ZPhaseConcurrentMark); - ZHeap::heap()->mark(); + ZHeap::heap()->mark(true /* initial */); } // Phase 3: Pause Mark End @@ -345,7 +345,7 @@ while (!vm_operation(&cl)) { // Phase 3.5: Concurrent Mark Continue ZStatTimer timer(ZPhaseConcurrentMarkContinue); - ZHeap::heap()->mark(); + ZHeap::heap()->mark(false /* initial */); } } diff -r 5a2af44ecb83 -r 3a168f782e80 src/hotspot/share/gc/z/zHeap.cpp --- a/src/hotspot/share/gc/z/zHeap.cpp Tue Oct 16 13:43:04 2018 +0200 +++ b/src/hotspot/share/gc/z/zHeap.cpp Tue Oct 16 13:14:18 2018 +0200 @@ -296,8 +296,8 @@ ZStatHeap::set_at_mark_start(capacity(), used()); } -void ZHeap::mark() { - _mark.mark(); +void ZHeap::mark(bool initial) { + _mark.mark(initial); } void ZHeap::mark_flush_and_free(Thread* thread) { diff -r 5a2af44ecb83 -r 3a168f782e80 src/hotspot/share/gc/z/zHeap.hpp --- a/src/hotspot/share/gc/z/zHeap.hpp Tue Oct 16 13:43:04 2018 +0200 +++ b/src/hotspot/share/gc/z/zHeap.hpp Tue Oct 16 13:14:18 2018 +0200 @@ -133,7 +133,7 @@ bool is_object_strongly_live(uintptr_t addr) const; template void mark_object(uintptr_t addr); void mark_start(); - void mark(); + void mark(bool initial); void mark_flush_and_free(Thread* thread); bool mark_end(); diff -r 5a2af44ecb83 -r 3a168f782e80 src/hotspot/share/gc/z/zHeapIterator.cpp --- a/src/hotspot/share/gc/z/zHeapIterator.cpp Tue Oct 16 13:43:04 2018 +0200 +++ b/src/hotspot/share/gc/z/zHeapIterator.cpp Tue Oct 16 13:14:18 2018 +0200 @@ -172,8 +172,10 @@ // this the user would have expected to see ObjectFree events for // unreachable objects in the tag map. ZRootsIterator roots; + ZConcurrentRootsIterator concurrent_roots(false /* marking */); ZHeapIteratorRootOopClosure root_cl(this); roots.oops_do(&root_cl, true /* visit_jvmti_weak_export */); + concurrent_roots.oops_do(&root_cl); } // Drain stack diff -r 5a2af44ecb83 -r 3a168f782e80 src/hotspot/share/gc/z/zMark.cpp --- a/src/hotspot/share/gc/z/zMark.cpp Tue Oct 16 13:43:04 2018 +0200 +++ b/src/hotspot/share/gc/z/zMark.cpp Tue Oct 16 13:14:18 2018 +0200 @@ -615,6 +615,34 @@ stacks->free(&_allocator); } +class ZMarkConcurrentRootsIteratorClosure : public ZRootsIteratorClosure { +public: + virtual void do_oop(oop* p) { + ZBarrier::mark_barrier_on_oop_field(p, false /* finalizable */); + } + + virtual void do_oop(narrowOop* p) { + ShouldNotReachHere(); + } +}; + + +class ZMarkConcurrentRootsTask : public ZTask { +private: + ZConcurrentRootsIterator _roots; + ZMarkConcurrentRootsIteratorClosure _cl; + +public: + ZMarkConcurrentRootsTask(ZMark* mark) : + ZTask("ZMarkConcurrentRootsTask"), + _roots(true /* marking */), + _cl() {} + + virtual void work() { + _roots.oops_do(&_cl); + } +}; + class ZMarkTask : public ZTask { private: ZMark* const _mark; @@ -637,7 +665,12 @@ } }; -void ZMark::mark() { +void ZMark::mark(bool initial) { + if (initial) { + ZMarkConcurrentRootsTask task(this); + _workers->run_concurrent(&task); + } + ZMarkTask task(this); _workers->run_concurrent(&task); } diff -r 5a2af44ecb83 -r 3a168f782e80 src/hotspot/share/gc/z/zMark.hpp --- a/src/hotspot/share/gc/z/zMark.hpp Tue Oct 16 13:43:04 2018 +0200 +++ b/src/hotspot/share/gc/z/zMark.hpp Tue Oct 16 13:14:18 2018 +0200 @@ -108,7 +108,7 @@ template void mark_object(uintptr_t addr); void start(); - void mark(); + void mark(bool initial); bool end(); void flush_and_free(); diff -r 5a2af44ecb83 -r 3a168f782e80 src/hotspot/share/gc/z/zRootsIterator.cpp --- a/src/hotspot/share/gc/z/zRootsIterator.cpp Tue Oct 16 13:43:04 2018 +0200 +++ b/src/hotspot/share/gc/z/zRootsIterator.cpp Tue Oct 16 13:14:18 2018 +0200 @@ -28,6 +28,7 @@ #include "code/codeCache.hpp" #include "compiler/oopMap.hpp" #include "gc/shared/oopStorageParState.inline.hpp" +#include "gc/shared/suspendibleThreadSet.hpp" #include "gc/z/zGlobals.hpp" #include "gc/z/zNMethodTable.hpp" #include "gc/z/zOopClosures.inline.hpp" @@ -52,16 +53,20 @@ static const ZStatSubPhase ZSubPhasePauseRoots("Pause Roots"); static const ZStatSubPhase ZSubPhasePauseRootsTeardown("Pause Roots Teardown"); static const ZStatSubPhase ZSubPhasePauseRootsUniverse("Pause Roots Universe"); -static const ZStatSubPhase ZSubPhasePauseRootsJNIHandles("Pause Roots JNIHandles"); static const ZStatSubPhase ZSubPhasePauseRootsObjectSynchronizer("Pause Roots ObjectSynchronizer"); static const ZStatSubPhase ZSubPhasePauseRootsManagement("Pause Roots Management"); static const ZStatSubPhase ZSubPhasePauseRootsJVMTIExport("Pause Roots JVMTIExport"); static const ZStatSubPhase ZSubPhasePauseRootsJVMTIWeakExport("Pause Roots JVMTIWeakExport"); static const ZStatSubPhase ZSubPhasePauseRootsSystemDictionary("Pause Roots SystemDictionary"); -static const ZStatSubPhase ZSubPhasePauseRootsClassLoaderDataGraph("Pause Roots ClassLoaderDataGraph"); static const ZStatSubPhase ZSubPhasePauseRootsThreads("Pause Roots Threads"); static const ZStatSubPhase ZSubPhasePauseRootsCodeCache("Pause Roots CodeCache"); +static const ZStatSubPhase ZSubPhaseConcurrentRootsSetup("Concurrent Roots Setup"); +static const ZStatSubPhase ZSubPhaseConcurrentRoots("Concurrent Roots"); +static const ZStatSubPhase ZSubPhaseConcurrentRootsTeardown("Concurrent Roots Teardown"); +static const ZStatSubPhase ZSubPhaseConcurrentRootsJNIHandles("Concurrent Roots JNIHandles"); +static const ZStatSubPhase ZSubPhaseConcurrentRootsClassLoaderDataGraph("Concurrent Roots ClassLoaderDataGraph"); + static const ZStatSubPhase ZSubPhasePauseWeakRootsSetup("Pause Weak Roots Setup"); static const ZStatSubPhase ZSubPhasePauseWeakRoots("Pause Weak Roots"); static const ZStatSubPhase ZSubPhasePauseWeakRootsTeardown("Pause Weak Roots Teardown"); @@ -128,21 +133,17 @@ } ZRootsIterator::ZRootsIterator() : - _jni_handles_iter(JNIHandles::global_handles()), _universe(this), _object_synchronizer(this), _management(this), _jvmti_export(this), _jvmti_weak_export(this), _system_dictionary(this), - _jni_handles(this), - _class_loader_data_graph(this), _threads(this), _code_cache(this) { assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); ZStatTimer timer(ZSubPhasePauseRootsSetup); Threads::change_thread_claim_parity(); - ClassLoaderDataGraph::clear_claimed_marks(); COMPILER2_PRESENT(DerivedPointerTable::clear()); CodeCache::gc_prologue(); ZNMethodTable::gc_prologue(); @@ -163,11 +164,6 @@ Universe::oops_do(cl); } -void ZRootsIterator::do_jni_handles(ZRootsIteratorClosure* cl) { - ZStatTimer timer(ZSubPhasePauseRootsJNIHandles); - _jni_handles_iter.oops_do(cl); -} - void ZRootsIterator::do_object_synchronizer(ZRootsIteratorClosure* cl) { ZStatTimer timer(ZSubPhasePauseRootsObjectSynchronizer); ObjectSynchronizer::oops_do(cl); @@ -194,12 +190,6 @@ SystemDictionary::oops_do(cl); } -void ZRootsIterator::do_class_loader_data_graph(ZRootsIteratorClosure* cl) { - ZStatTimer timer(ZSubPhasePauseRootsClassLoaderDataGraph); - CLDToOopClosure cld_cl(cl); - ClassLoaderDataGraph::cld_do(&cld_cl); -} - void ZRootsIterator::do_threads(ZRootsIteratorClosure* cl) { ZStatTimer timer(ZSubPhasePauseRootsThreads); ResourceMark rm; @@ -218,8 +208,6 @@ _management.oops_do(cl); _jvmti_export.oops_do(cl); _system_dictionary.oops_do(cl); - _jni_handles.oops_do(cl); - _class_loader_data_graph.oops_do(cl); _threads.oops_do(cl); _code_cache.oops_do(cl); if (visit_jvmti_weak_export) { @@ -227,6 +215,43 @@ } } +ZConcurrentRootsIterator::ZConcurrentRootsIterator(bool marking) : + _marking(marking), + _sts_joiner(marking /* active */), + _jni_handles_iter(JNIHandles::global_handles()), + _jni_handles(this), + _class_loader_data_graph(this) { + ZStatTimer timer(ZSubPhaseConcurrentRootsSetup); + if (_marking) { + ClassLoaderDataGraph_lock->lock(); + ClassLoaderDataGraph::clear_claimed_marks(); + } +} + +ZConcurrentRootsIterator::~ZConcurrentRootsIterator() { + ZStatTimer timer(ZSubPhaseConcurrentRootsTeardown); + if (_marking) { + ClassLoaderDataGraph_lock->unlock(); + } +} + +void ZConcurrentRootsIterator::do_jni_handles(ZRootsIteratorClosure* cl) { + ZStatTimer timer(ZSubPhaseConcurrentRootsJNIHandles); + _jni_handles_iter.oops_do(cl); +} + +void ZConcurrentRootsIterator::do_class_loader_data_graph(ZRootsIteratorClosure* cl) { + ZStatTimer timer(ZSubPhaseConcurrentRootsClassLoaderDataGraph); + CLDToOopClosure cld_cl(cl, _marking /* must_claim */); + ClassLoaderDataGraph::cld_do(&cld_cl); +} + +void ZConcurrentRootsIterator::oops_do(ZRootsIteratorClosure* cl) { + ZStatTimer timer(ZSubPhaseConcurrentRoots); + _jni_handles.oops_do(cl); + _class_loader_data_graph.oops_do(cl); +} + ZWeakRootsIterator::ZWeakRootsIterator() : _jvmti_weak_export(this), _jfr_weak(this) { diff -r 5a2af44ecb83 -r 3a168f782e80 src/hotspot/share/gc/z/zRootsIterator.hpp --- a/src/hotspot/share/gc/z/zRootsIterator.hpp Tue Oct 16 13:43:04 2018 +0200 +++ b/src/hotspot/share/gc/z/zRootsIterator.hpp Tue Oct 16 13:14:18 2018 +0200 @@ -25,6 +25,7 @@ #define SHARE_GC_Z_ZROOTSITERATOR_HPP #include "gc/shared/oopStorageParState.hpp" +#include "gc/shared/suspendibleThreadSet.hpp" #include "memory/allocation.hpp" #include "memory/iterator.hpp" #include "runtime/thread.hpp" @@ -37,8 +38,7 @@ } }; -typedef OopStorage::ParState ZOopStorageIterator; -typedef OopStorage::ParState ZConcurrentOopStorageIterator; +typedef OopStorage::ParState ZOopStorageIterator; template class ZSerialOopsDo { @@ -86,29 +86,23 @@ class ZRootsIterator { private: - ZOopStorageIterator _jni_handles_iter; - void do_universe(ZRootsIteratorClosure* cl); - void do_jni_handles(ZRootsIteratorClosure* cl); void do_object_synchronizer(ZRootsIteratorClosure* cl); void do_management(ZRootsIteratorClosure* cl); void do_jvmti_export(ZRootsIteratorClosure* cl); void do_jvmti_weak_export(ZRootsIteratorClosure* cl); void do_system_dictionary(ZRootsIteratorClosure* cl); - void do_class_loader_data_graph(ZRootsIteratorClosure* cl); void do_threads(ZRootsIteratorClosure* cl); void do_code_cache(ZRootsIteratorClosure* cl); - ZSerialOopsDo _universe; - ZSerialOopsDo _object_synchronizer; - ZSerialOopsDo _management; - ZSerialOopsDo _jvmti_export; - ZSerialOopsDo _jvmti_weak_export; - ZSerialOopsDo _system_dictionary; - ZParallelOopsDo _jni_handles; - ZParallelOopsDo _class_loader_data_graph; - ZParallelOopsDo _threads; - ZParallelOopsDo _code_cache; + ZSerialOopsDo _universe; + ZSerialOopsDo _object_synchronizer; + ZSerialOopsDo _management; + ZSerialOopsDo _jvmti_export; + ZSerialOopsDo _jvmti_weak_export; + ZSerialOopsDo _system_dictionary; + ZParallelOopsDo _threads; + ZParallelOopsDo _code_cache; public: ZRootsIterator(); @@ -117,6 +111,25 @@ void oops_do(ZRootsIteratorClosure* cl, bool visit_jvmti_weak_export = false); }; +class ZConcurrentRootsIterator { +private: + const bool _marking; + SuspendibleThreadSetJoiner _sts_joiner; + ZOopStorageIterator _jni_handles_iter; + + void do_jni_handles(ZRootsIteratorClosure* cl); + void do_class_loader_data_graph(ZRootsIteratorClosure* cl); + + ZParallelOopsDo _jni_handles; + ZParallelOopsDo _class_loader_data_graph; + +public: + ZConcurrentRootsIterator(bool marking); + ~ZConcurrentRootsIterator(); + + void oops_do(ZRootsIteratorClosure* cl); +}; + class ZWeakRootsIterator { private: void do_jvmti_weak_export(BoolObjectClosure* is_alive, ZRootsIteratorClosure* cl); @@ -135,9 +148,9 @@ class ZConcurrentWeakRootsIterator { private: - ZConcurrentOopStorageIterator _vm_weak_handles_iter; - ZConcurrentOopStorageIterator _jni_weak_handles_iter; - ZConcurrentOopStorageIterator _string_table_iter; + ZOopStorageIterator _vm_weak_handles_iter; + ZOopStorageIterator _jni_weak_handles_iter; + ZOopStorageIterator _string_table_iter; void do_vm_weak_handles(ZRootsIteratorClosure* cl); void do_jni_weak_handles(ZRootsIteratorClosure* cl); diff -r 5a2af44ecb83 -r 3a168f782e80 src/hotspot/share/runtime/mutexLocker.cpp --- a/src/hotspot/share/runtime/mutexLocker.cpp Tue Oct 16 13:43:04 2018 +0200 +++ b/src/hotspot/share/runtime/mutexLocker.cpp Tue Oct 16 13:14:18 2018 +0200 @@ -166,6 +166,16 @@ fatal("must own lock %s", lock->name()); } +// a weaker assertion than the above +void assert_locked_or_safepoint_weak(const Monitor * lock) { + if (IgnoreLockingAssertions) return; + assert(lock != NULL, "Need non-NULL lock"); + if (lock->is_locked()) return; + if (SafepointSynchronize::is_at_safepoint()) return; + if (!Universe::is_fully_initialized()) return; + fatal("must own lock %s", lock->name()); +} + // a stronger assertion than the above void assert_lock_strong(const Monitor * lock) { if (IgnoreLockingAssertions) return; diff -r 5a2af44ecb83 -r 3a168f782e80 src/hotspot/share/runtime/mutexLocker.hpp --- a/src/hotspot/share/runtime/mutexLocker.hpp Tue Oct 16 13:43:04 2018 +0200 +++ b/src/hotspot/share/runtime/mutexLocker.hpp Tue Oct 16 13:14:18 2018 +0200 @@ -199,9 +199,11 @@ // for debugging: check that we're already owning this lock (or are at a safepoint) #ifdef ASSERT void assert_locked_or_safepoint(const Monitor * lock); +void assert_locked_or_safepoint_weak(const Monitor * lock); void assert_lock_strong(const Monitor * lock); #else #define assert_locked_or_safepoint(lock) +#define assert_locked_or_safepoint_weak(lock) #define assert_lock_strong(lock) #endif