8226311: Shenandoah: Concurrent evacuation of OopStorage backed weak roots
Reviewed-by: rkennke
--- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp Mon Jun 24 15:19:11 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp Mon Jun 24 11:46:46 2019 -0400
@@ -31,6 +31,7 @@
#include "gc/shared/gcTimer.hpp"
#include "gc/shared/referenceProcessor.hpp"
#include "gc/shared/referenceProcessorPhaseTimes.hpp"
+#include "gc/shared/strongRootsScope.hpp"
#include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
#include "gc/shenandoah/shenandoahClosures.inline.hpp"
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp Mon Jun 24 15:19:11 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp Mon Jun 24 11:46:46 2019 -0400
@@ -1523,7 +1523,8 @@
if (ShenandoahVerify) {
if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
- verifier()->verify_roots_no_forwarded_except(ShenandoahRootVerifier::JNIHandleRoots);
+ ShenandoahRootVerifier::RootTypes types = ShenandoahRootVerifier::combine(ShenandoahRootVerifier::JNIHandleRoots, ShenandoahRootVerifier::WeakRoots);
+ verifier()->verify_roots_no_forwarded_except(types);
} else {
verifier()->verify_roots_no_forwarded();
}
@@ -1590,6 +1591,7 @@
class ShenandoahConcurrentRootsEvacUpdateTask : public AbstractGangTask {
private:
ShenandoahJNIHandleRoots<true /*concurrent*/> _jni_roots;
+ ShenandoahWeakRoots<true /*concurrent*/> _weak_roots;
public:
ShenandoahConcurrentRootsEvacUpdateTask() :
@@ -1599,7 +1601,9 @@
void work(uint worker_id) {
ShenandoahEvacOOMScope oom;
ShenandoahEvacuateUpdateRootsClosure cl;
+
_jni_roots.oops_do<ShenandoahEvacuateUpdateRootsClosure>(&cl);
+ _weak_roots.oops_do<ShenandoahEvacuateUpdateRootsClosure>(&cl);
}
};
--- a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp Mon Jun 24 15:19:11 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp Mon Jun 24 11:46:46 2019 -0400
@@ -27,16 +27,13 @@
#include "classfile/stringTable.hpp"
#include "classfile/systemDictionary.hpp"
#include "code/codeCache.hpp"
-#include "gc/shenandoah/shenandoahClosures.inline.hpp"
#include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
#include "gc/shenandoah/shenandoahHeap.hpp"
-#include "gc/shenandoah/shenandoahHeuristics.hpp"
#include "gc/shenandoah/shenandoahPhaseTimings.hpp"
#include "gc/shenandoah/shenandoahStringDedup.hpp"
#include "gc/shenandoah/shenandoahTimingTracker.hpp"
#include "gc/shenandoah/shenandoahVMOperations.hpp"
-#include "gc/shared/weakProcessor.inline.hpp"
-#include "memory/allocation.inline.hpp"
+#include "jfr/jfr.hpp"
#include "memory/iterator.hpp"
#include "memory/resourceArea.hpp"
#include "memory/universe.hpp"
@@ -71,6 +68,40 @@
_jvmti_root.oops_do(cl, worker_id);
}
+ShenandoahWeakSerialRoot::ShenandoahWeakSerialRoot(ShenandoahWeakSerialRoot::WeakOopsDo weak_oops_do, ShenandoahPhaseTimings::GCParPhases phase) :
+ _weak_oops_do(weak_oops_do), _phase(phase) {
+}
+
+void ShenandoahWeakSerialRoot::weak_oops_do(BoolObjectClosure* is_alive, OopClosure* keep_alive, uint worker_id) {
+ if (_claimed.try_set()) {
+ ShenandoahWorkerTimings* worker_times = ShenandoahHeap::heap()->phase_timings()->worker_times();
+ ShenandoahWorkerTimingsTracker timer(worker_times, _phase, worker_id);
+ _weak_oops_do(is_alive, keep_alive);
+ }
+}
+
+#if INCLUDE_JVMTI
+ShenandoahJVMTIWeakRoot::ShenandoahJVMTIWeakRoot() :
+ ShenandoahWeakSerialRoot(&JvmtiExport::weak_oops_do, ShenandoahPhaseTimings::JVMTIWeakRoots) {
+}
+#endif // INCLUDE_JVMTI
+
+#if INCLUDE_JFR
+ShenandoahJFRWeakRoot::ShenandoahJFRWeakRoot() :
+ ShenandoahWeakSerialRoot(&Jfr::weak_oops_do, ShenandoahPhaseTimings::JFRWeakRoots) {
+}
+#endif // INCLUDE_JFR
+
+void ShenandoahSerialWeakRoots::weak_oops_do(BoolObjectClosure* is_alive, OopClosure* keep_alive, uint worker_id) {
+ JVMTI_ONLY(_jvmti_weak_roots.weak_oops_do(is_alive, keep_alive, worker_id);)
+ JFR_ONLY(_jfr_weak_roots.weak_oops_do(is_alive, keep_alive, worker_id);)
+}
+
+void ShenandoahSerialWeakRoots::weak_oops_do(OopClosure* cl, uint worker_id) {
+ AlwaysTrueClosure always_true;
+ weak_oops_do(&always_true, cl, worker_id);
+}
+
ShenandoahThreadRoots::ShenandoahThreadRoots(bool is_par) : _is_par(is_par) {
Threads::change_thread_claim_token();
}
@@ -93,17 +124,6 @@
Threads::assert_all_threads_claimed();
}
-ShenandoahWeakRoots::ShenandoahWeakRoots(uint n_workers) :
- _process_timings(n_workers),
- _task(&_process_timings, n_workers) {
-}
-
-ShenandoahWeakRoots::~ShenandoahWeakRoots() {
- ShenandoahWorkerTimings* worker_times = ShenandoahHeap::heap()->phase_timings()->worker_times();
- ShenandoahTimingConverter::weak_processing_timing_to_shenandoah_timing(&_process_timings,
- worker_times);
-}
-
ShenandoahStringDedupRoots::ShenandoahStringDedupRoots() {
if (ShenandoahStringDedup::is_enabled()) {
StringDedup::gc_prologue(false);
@@ -137,7 +157,6 @@
ShenandoahRootEvacuator::ShenandoahRootEvacuator(uint n_workers, ShenandoahPhaseTimings::Phase phase, bool include_concurrent_roots) :
ShenandoahRootProcessor(phase),
_thread_roots(n_workers > 1),
- _weak_roots(n_workers),
_include_concurrent_roots(include_concurrent_roots) {
}
@@ -147,29 +166,28 @@
AlwaysTrueClosure always_true;
_serial_roots.oops_do(oops, worker_id);
+ _serial_weak_roots.weak_oops_do(oops, worker_id);
if (_include_concurrent_roots) {
_jni_roots.oops_do<OopClosure>(oops, worker_id);
+ _weak_roots.oops_do<OopClosure>(oops, worker_id);
}
_thread_roots.oops_do(oops, NULL, worker_id);
_cld_roots.cld_do(&clds, worker_id);
_code_roots.code_blobs_do(&blobsCl, worker_id);
- _weak_roots.oops_do<AlwaysTrueClosure, OopClosure>(&always_true, oops, worker_id);
_dedup_roots.oops_do(&always_true, oops, worker_id);
}
ShenandoahRootUpdater::ShenandoahRootUpdater(uint n_workers, ShenandoahPhaseTimings::Phase phase, bool update_code_cache) :
ShenandoahRootProcessor(phase),
_thread_roots(n_workers > 1),
- _weak_roots(n_workers),
_update_code_cache(update_code_cache) {
}
ShenandoahRootAdjuster::ShenandoahRootAdjuster(uint n_workers, ShenandoahPhaseTimings::Phase phase) :
ShenandoahRootProcessor(phase),
- _thread_roots(n_workers > 1),
- _weak_roots(n_workers) {
+ _thread_roots(n_workers > 1) {
assert(ShenandoahHeap::heap()->is_full_gc_in_progress(), "Full GC only");
}
@@ -185,7 +203,8 @@
_cld_roots.cld_do(&adjust_cld_closure, worker_id);
_code_roots.code_blobs_do(&adjust_code_closure, worker_id);
- _weak_roots.oops_do<AlwaysTrueClosure, OopClosure>(&always_true, oops, worker_id);
+ _serial_weak_roots.weak_oops_do(oops, worker_id);
+ _weak_roots.oops_do<OopClosure>(oops, worker_id);
_dedup_roots.oops_do(&always_true, oops, worker_id);
}
--- a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp Mon Jun 24 15:19:11 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp Mon Jun 24 11:46:46 2019 -0400
@@ -29,11 +29,7 @@
#include "gc/shenandoah/shenandoahCodeRoots.hpp"
#include "gc/shenandoah/shenandoahHeap.hpp"
#include "gc/shenandoah/shenandoahPhaseTimings.hpp"
-#include "gc/shared/strongRootsScope.hpp"
-#include "gc/shared/weakProcessor.hpp"
-#include "gc/shared/weakProcessorPhaseTimes.hpp"
-#include "gc/shared/workgroup.hpp"
-#include "memory/allocation.hpp"
+#include "gc/shenandoah/shenandoahSharedVariables.hpp"
#include "memory/iterator.hpp"
class ShenandoahSerialRoot {
@@ -61,10 +57,102 @@
void oops_do(OopClosure* cl, uint worker_id);
};
+class ShenandoahWeakSerialRoot {
+ typedef void (*WeakOopsDo)(BoolObjectClosure*, OopClosure*);
+private:
+ ShenandoahSharedFlag _claimed;
+ const WeakOopsDo _weak_oops_do;
+ const ShenandoahPhaseTimings::GCParPhases _phase;
+
+public:
+ ShenandoahWeakSerialRoot(WeakOopsDo oops_do, ShenandoahPhaseTimings::GCParPhases);
+ void weak_oops_do(BoolObjectClosure* is_alive, OopClosure* keep_alive, uint worker_id);
+};
+
+#if INCLUDE_JVMTI
+class ShenandoahJVMTIWeakRoot : public ShenandoahWeakSerialRoot {
+public:
+ ShenandoahJVMTIWeakRoot();
+};
+#endif // INCLUDE_JVMTI
+
+#if INCLUDE_JFR
+class ShenandoahJFRWeakRoot : public ShenandoahWeakSerialRoot {
+public:
+ ShenandoahJFRWeakRoot();
+};
+#endif // INCLUDE_JFR
+
+class ShenandoahSerialWeakRoots {
+private:
+ JVMTI_ONLY(ShenandoahJVMTIWeakRoot _jvmti_weak_roots;)
+ JFR_ONLY(ShenandoahJFRWeakRoot _jfr_weak_roots;)
+public:
+ void weak_oops_do(BoolObjectClosure* is_alive, OopClosure* keep_alive, uint worker_id);
+ void weak_oops_do(OopClosure* cl, uint worker_id);
+};
+
+template <bool CONCURRENT>
+class ShenandoahWeakRoot {
+private:
+ OopStorage::ParState<CONCURRENT, false /* is_const */> _itr;
+ const ShenandoahPhaseTimings::GCParPhases _phase;
+public:
+ ShenandoahWeakRoot(OopStorage* storage, ShenandoahPhaseTimings::GCParPhases phase);
+
+ template <typename Closure>
+ void oops_do(Closure* cl, uint worker_id);
+};
+
+template <>
+class ShenandoahWeakRoot<false /*concurrent*/> {
+private:
+ OopStorage::ParState<false /*concurrent*/, false /*is_const*/> _itr;
+ const ShenandoahPhaseTimings::GCParPhases _phase;
+
+public:
+ ShenandoahWeakRoot(OopStorage* storage, ShenandoahPhaseTimings::GCParPhases phase);
+
+ template <typename IsAliveClosure, typename KeepAliveClosure>
+ void weak_oops_do(IsAliveClosure* is_alive, KeepAliveClosure* keep_alive, uint worker_id);
+};
+
+template <bool CONCURRENT>
+class ShenandoahWeakRoots {
+private:
+ ShenandoahWeakRoot<CONCURRENT> _jni_roots;
+ ShenandoahWeakRoot<CONCURRENT> _string_table_roots;
+ ShenandoahWeakRoot<CONCURRENT> _resolved_method_table_roots;
+ ShenandoahWeakRoot<CONCURRENT> _vm_roots;
+
+public:
+ ShenandoahWeakRoots();
+
+ template <typename Closure>
+ void oops_do(Closure* cl, uint worker_id = 0);
+};
+
+template <>
+class ShenandoahWeakRoots<false /*concurrent */> {
+private:
+ ShenandoahWeakRoot<false /*concurrent*/> _jni_roots;
+ ShenandoahWeakRoot<false /*concurrent*/> _string_table_roots;
+ ShenandoahWeakRoot<false /*concurrent*/> _resolved_method_table_roots;
+ ShenandoahWeakRoot<false /*concurrent*/> _vm_roots;
+public:
+ ShenandoahWeakRoots();
+
+ template <typename Closure>
+ void oops_do(Closure* cl, uint worker_id = 0);
+
+ template <typename IsAliveClosure, typename KeepAliveClosure>
+ void weak_oops_do(IsAliveClosure* is_alive, KeepAliveClosure* keep_alive, uint worker_id);
+};
+
template <bool CONCURRENT>
class ShenandoahJNIHandleRoots {
private:
- OopStorage::ParState<CONCURRENT, false /* is_const */> _itr;
+ OopStorage::ParState<CONCURRENT, false /*is_const*/> _itr;
public:
ShenandoahJNIHandleRoots();
@@ -83,18 +171,6 @@
void threads_do(ThreadClosure* tc, uint worker_id);
};
-class ShenandoahWeakRoots {
-private:
- WeakProcessorPhaseTimes _process_timings;
- WeakProcessor::Task _task;
-public:
- ShenandoahWeakRoots(uint n_workers);
- ~ShenandoahWeakRoots();
-
- template <typename IsAlive, typename KeepAlive>
- void oops_do(IsAlive* is_alive, KeepAlive* keep_alive, uint worker_id);
-};
-
class ShenandoahStringDedupRoots {
public:
ShenandoahStringDedupRoots();
@@ -183,7 +259,8 @@
ShenandoahJNIHandleRoots<false /*concurrent*/> _jni_roots;
ShenandoahClassLoaderDataRoots<false /*single threaded*/> _cld_roots;
ShenandoahThreadRoots _thread_roots;
- ShenandoahWeakRoots _weak_roots;
+ ShenandoahSerialWeakRoots _serial_weak_roots;
+ ShenandoahWeakRoots<false /*concurrent*/> _weak_roots;
ShenandoahStringDedupRoots _dedup_roots;
ShenandoahCodeCacheRoots<ShenandoahCsetCodeRootsIterator> _code_roots;
bool _include_concurrent_roots;
@@ -201,7 +278,8 @@
ShenandoahJNIHandleRoots<false /*concurrent*/> _jni_roots;
ShenandoahClassLoaderDataRoots<false /*single threaded*/> _cld_roots;
ShenandoahThreadRoots _thread_roots;
- ShenandoahWeakRoots _weak_roots;
+ ShenandoahSerialWeakRoots _serial_weak_roots;
+ ShenandoahWeakRoots<false /*concurrent*/> _weak_roots;
ShenandoahStringDedupRoots _dedup_roots;
ShenandoahCodeCacheRoots<ShenandoahCsetCodeRootsIterator> _code_roots;
const bool _update_code_cache;
@@ -220,7 +298,8 @@
ShenandoahJNIHandleRoots<false /*concurrent*/> _jni_roots;
ShenandoahClassLoaderDataRoots<false /*single threaded*/> _cld_roots;
ShenandoahThreadRoots _thread_roots;
- ShenandoahWeakRoots _weak_roots;
+ ShenandoahSerialWeakRoots _serial_weak_roots;
+ ShenandoahWeakRoots<false /*concurrent*/> _weak_roots;
ShenandoahStringDedupRoots _dedup_roots;
ShenandoahCodeCacheRoots<ShenandoahAllCodeRootsIterator> _code_roots;
--- a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.inline.hpp Mon Jun 24 15:19:11 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.inline.hpp Mon Jun 24 11:46:46 2019 -0400
@@ -25,15 +25,84 @@
#define SHARE_GC_SHENANDOAH_SHENANDOAHROOTPROCESSOR_INLINE_HPP
#include "classfile/classLoaderDataGraph.hpp"
+#include "classfile/stringTable.hpp"
+#include "classfile/systemDictionary.hpp"
#include "gc/shared/oopStorageParState.inline.hpp"
#include "gc/shenandoah/shenandoahHeuristics.hpp"
#include "gc/shenandoah/shenandoahRootProcessor.hpp"
#include "gc/shenandoah/shenandoahTimingTracker.hpp"
#include "gc/shenandoah/shenandoahUtils.hpp"
#include "memory/resourceArea.hpp"
+#include "prims/resolvedMethodTable.hpp"
#include "runtime/safepoint.hpp"
template <bool CONCURRENT>
+inline ShenandoahWeakRoot<CONCURRENT>::ShenandoahWeakRoot(OopStorage* storage, ShenandoahPhaseTimings::GCParPhases phase) :
+ _itr(storage), _phase(phase) {
+}
+
+template <bool CONCURRENT>
+template <typename Closure>
+inline void ShenandoahWeakRoot<CONCURRENT>::oops_do(Closure* cl, uint worker_id) {
+ if (CONCURRENT) {
+ _itr.oops_do(cl);
+ } else {
+ ShenandoahWorkerTimings* worker_times = ShenandoahHeap::heap()->phase_timings()->worker_times();
+ ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::ThreadRoots, worker_id);
+ _itr.oops_do(cl);
+ }
+}
+
+inline ShenandoahWeakRoot<false>::ShenandoahWeakRoot(OopStorage* storage, ShenandoahPhaseTimings::GCParPhases phase) :
+ _itr(storage), _phase(phase) {
+}
+
+template <typename IsAliveClosure, typename KeepAliveClosure>
+void ShenandoahWeakRoot<false /* concurrent */>::weak_oops_do(IsAliveClosure* is_alive, KeepAliveClosure* keep_alive, uint worker_id) {
+ ShenandoahWorkerTimings* worker_times = ShenandoahHeap::heap()->phase_timings()->worker_times();
+ ShenandoahWorkerTimingsTracker timer(worker_times, _phase, worker_id);
+ _itr.weak_oops_do(is_alive, keep_alive);
+}
+
+template <bool CONCURRENT>
+ShenandoahWeakRoots<CONCURRENT>::ShenandoahWeakRoots() :
+ _jni_roots(JNIHandles::weak_global_handles(), ShenandoahPhaseTimings::JNIWeakRoots),
+ _string_table_roots(StringTable::weak_storage(), ShenandoahPhaseTimings::StringTableRoots),
+ _resolved_method_table_roots(ResolvedMethodTable::weak_storage(), ShenandoahPhaseTimings::ResolvedMethodTableRoots),
+ _vm_roots(SystemDictionary::vm_weak_oop_storage(), ShenandoahPhaseTimings::VMWeakRoots) {
+}
+
+template <bool CONCURRENT>
+template <typename Closure>
+void ShenandoahWeakRoots<CONCURRENT>::oops_do(Closure* cl, uint worker_id) {
+ _jni_roots.oops_do(cl, worker_id);
+ _string_table_roots.oops_do(cl, worker_id);
+ _resolved_method_table_roots.oops_do(cl, worker_id);
+ _vm_roots.oops_do(cl, worker_id);
+}
+
+inline ShenandoahWeakRoots<false /* concurrent */>::ShenandoahWeakRoots() :
+ _jni_roots(JNIHandles::weak_global_handles(), ShenandoahPhaseTimings::JNIWeakRoots),
+ _string_table_roots(StringTable::weak_storage(), ShenandoahPhaseTimings::StringTableRoots),
+ _resolved_method_table_roots(ResolvedMethodTable::weak_storage(), ShenandoahPhaseTimings::ResolvedMethodTableRoots),
+ _vm_roots(SystemDictionary::vm_weak_oop_storage(), ShenandoahPhaseTimings::VMWeakRoots) {
+}
+
+template <typename IsAliveClosure, typename KeepAliveClosure>
+void ShenandoahWeakRoots<false /* concurrent*/>::weak_oops_do(IsAliveClosure* is_alive, KeepAliveClosure* keep_alive, uint worker_id) {
+ _jni_roots.weak_oops_do(is_alive, keep_alive, worker_id);
+ _string_table_roots.weak_oops_do(is_alive, keep_alive, worker_id);
+ _resolved_method_table_roots.weak_oops_do(is_alive, keep_alive, worker_id);
+ _vm_roots.weak_oops_do(is_alive, keep_alive, worker_id);
+}
+
+template <typename Closure>
+void ShenandoahWeakRoots<false /* concurrent */>::oops_do(Closure* cl, uint worker_id) {
+ AlwaysTrueClosure always_true;
+ weak_oops_do<AlwaysTrueClosure, Closure>(&always_true, cl, worker_id);
+}
+
+template <bool CONCURRENT>
ShenandoahJNIHandleRoots<CONCURRENT>::ShenandoahJNIHandleRoots() :
_itr(JNIHandles::global_handles()) {
}
@@ -50,11 +119,6 @@
}
}
-template <typename IsAlive, typename KeepAlive>
-void ShenandoahWeakRoots::oops_do(IsAlive* is_alive, KeepAlive* keep_alive, uint worker_id) {
- _task.work<IsAlive, KeepAlive>(worker_id, is_alive, keep_alive);
-}
-
template <bool SINGLE_THREADED>
ShenandoahClassLoaderDataRoots<SINGLE_THREADED>::ShenandoahClassLoaderDataRoots() {
if (!SINGLE_THREADED) {
@@ -197,7 +261,8 @@
_code_roots.code_blobs_do(&update_blobs, worker_id);
}
- _weak_roots.oops_do<IsAlive, KeepAlive>(is_alive, keep_alive, worker_id);
+ _serial_weak_roots.weak_oops_do(is_alive, keep_alive, worker_id);
+ _weak_roots.weak_oops_do(is_alive, keep_alive, worker_id);
_dedup_roots.oops_do(is_alive, keep_alive, worker_id);
}
--- a/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp Mon Jun 24 15:19:11 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp Mon Jun 24 11:46:46 2019 -0400
@@ -54,6 +54,10 @@
return (_types & type) != 0;
}
+ShenandoahRootVerifier::RootTypes ShenandoahRootVerifier::combine(RootTypes t1, RootTypes t2) {
+ return static_cast<ShenandoahRootVerifier::RootTypes>(static_cast<uint>(t1) | static_cast<uint>(t2));
+}
+
void ShenandoahRootVerifier::oops_do(OopClosure* oops) {
CodeBlobToOopClosure blobs(oops, !CodeBlobToOopClosure::FixRelocations);
if (verify(CodeRoots)) {
--- a/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp Mon Jun 24 15:19:11 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp Mon Jun 24 11:46:46 2019 -0400
@@ -52,6 +52,8 @@
// Used to seed ShenandoahVerifier, do not honor root type filter
void roots_do(OopClosure* cl);
void strong_roots_do(OopClosure* cl);
+
+ static RootTypes combine(RootTypes t1, RootTypes t2);
private:
bool verify(RootTypes type) const;
};