8229451: ZGC: Make some roots invisible to the heap iterator
Reviewed-by: eosterlund
--- a/src/hotspot/share/gc/shared/gcThreadLocalData.hpp Fri Aug 23 08:48:18 2019 +0200
+++ b/src/hotspot/share/gc/shared/gcThreadLocalData.hpp Fri Aug 23 08:48:23 2019 +0200
@@ -40,6 +40,6 @@
// should consider placing frequently accessed fields first in
// T, so that field offsets relative to Thread are small, which
// often allows for a more compact instruction encoding.
-typedef uint64_t GCThreadLocalData[18]; // 144 bytes
+typedef uint64_t GCThreadLocalData[19]; // 152 bytes
#endif // SHARE_GC_SHARED_GCTHREADLOCALDATA_HPP
--- a/src/hotspot/share/gc/z/zHeapIterator.cpp Fri Aug 23 08:48:18 2019 +0200
+++ b/src/hotspot/share/gc/z/zHeapIterator.cpp Fri Aug 23 08:48:23 2019 +0200
@@ -193,7 +193,7 @@
ZStatTimerDisable disable;
// Push roots to visit
- push_roots<ZRootsIterator, false /* Concurrent */, false /* Weak */>();
+ push_roots<ZRootsIteratorNoInvisible, false /* Concurrent */, false /* Weak */>();
push_roots<ZConcurrentRootsIteratorClaimOther, true /* Concurrent */, false /* Weak */>();
if (VisitWeaks) {
push_roots<ZWeakRootsIterator, false /* Concurrent */, true /* Weak */>();
--- a/src/hotspot/share/gc/z/zMark.cpp Fri Aug 23 08:48:18 2019 +0200
+++ b/src/hotspot/share/gc/z/zMark.cpp Fri Aug 23 08:48:23 2019 +0200
@@ -156,7 +156,7 @@
ZMarkRootsTask(ZMark* mark) :
ZTask("ZMarkRootsTask"),
_mark(mark),
- _roots() {}
+ _roots(true /* visit_invisible */, false /* visit_jvmti_weak_export */) {}
virtual void work() {
_roots.oops_do(&_cl);
--- a/src/hotspot/share/gc/z/zObjArrayAllocator.cpp Fri Aug 23 08:48:18 2019 +0200
+++ b/src/hotspot/share/gc/z/zObjArrayAllocator.cpp Fri Aug 23 08:48:23 2019 +0200
@@ -22,6 +22,7 @@
*/
#include "precompiled.hpp"
+#include "gc/z/zThreadLocalData.hpp"
#include "gc/z/zObjArrayAllocator.hpp"
#include "gc/z/zUtils.inline.hpp"
#include "memory/universe.hpp"
@@ -57,8 +58,12 @@
_final_klass(klass) {}
oop ZObjArrayAllocator::finish(HeapWord* mem) const {
- HandleMark hm;
- Handle array(_thread, ObjArrayAllocator::finish(mem));
+ // Set mark word and initial klass pointer
+ ObjArrayAllocator::finish(mem);
+
+ // Keep the array alive across safepoints, but make it invisible
+ // to the heap itarator until the final klass pointer has been set
+ ZThreadLocalData::set_invisible_root(_thread, (oop*)&mem);
const size_t segment_max = ZUtils::bytes_to_words(64 * K);
const size_t skip = arrayOopDesc::header_size(ArrayKlass::cast(_klass)->element_type());
@@ -67,7 +72,7 @@
while (remaining > 0) {
// Clear segment
const size_t segment = MIN2(remaining, segment_max);
- Copy::zero_to_words((HeapWord*)array() + (_word_size - remaining), segment);
+ Copy::zero_to_words(mem + (_word_size - remaining), segment);
remaining -= segment;
if (remaining > 0) {
@@ -77,9 +82,12 @@
}
if (_klass != _final_klass) {
- // Set final klass
- oopDesc::release_set_klass((HeapWord*)array(), _final_klass);
+ // Set final klass pointer
+ oopDesc::release_set_klass(mem, _final_klass);
}
- return array();
+ // Make the array visible to the heap iterator
+ ZThreadLocalData::clear_invisible_root(_thread);
+
+ return oop(mem);
}
--- a/src/hotspot/share/gc/z/zRelocate.cpp Fri Aug 23 08:48:18 2019 +0200
+++ b/src/hotspot/share/gc/z/zRelocate.cpp Fri Aug 23 08:48:23 2019 +0200
@@ -69,12 +69,12 @@
public:
ZRelocateRootsTask() :
ZTask("ZRelocateRootsTask"),
- _roots() {}
+ _roots(true /* visit_invisible */, true /* visit_jvmti_weak_export */) {}
virtual void work() {
// During relocation we need to visit the JVMTI
// export weak roots to rehash the JVMTI tag map
- _roots.oops_do(&_cl, true /* visit_jvmti_weak_export */);
+ _roots.oops_do(&_cl);
}
};
--- a/src/hotspot/share/gc/z/zRootsIterator.cpp Fri Aug 23 08:48:18 2019 +0200
+++ b/src/hotspot/share/gc/z/zRootsIterator.cpp Fri Aug 23 08:48:23 2019 +0200
@@ -159,19 +159,26 @@
class ZRootsIteratorThreadClosure : public ThreadClosure {
private:
ZRootsIteratorClosure* _cl;
+ const bool _visit_invisible;
public:
- ZRootsIteratorThreadClosure(ZRootsIteratorClosure* cl) :
- _cl(cl) {}
+ ZRootsIteratorThreadClosure(ZRootsIteratorClosure* cl, bool visit_invisible) :
+ _cl(cl),
+ _visit_invisible(visit_invisible) {}
virtual void do_thread(Thread* thread) {
ZRootsIteratorCodeBlobClosure code_cl(_cl);
thread->oops_do(_cl, ClassUnloading ? &code_cl : NULL);
_cl->do_thread(thread);
+ if (_visit_invisible && ZThreadLocalData::has_invisible_root(thread)) {
+ _cl->do_oop(ZThreadLocalData::invisible_root(thread));
+ }
}
};
-ZRootsIterator::ZRootsIterator() :
+ZRootsIterator::ZRootsIterator(bool visit_invisible, bool visit_jvmti_weak_export) :
+ _visit_invisible(visit_invisible),
+ _visit_jvmti_weak_export(visit_jvmti_weak_export),
_universe(this),
_object_synchronizer(this),
_management(this),
@@ -239,7 +246,7 @@
void ZRootsIterator::do_threads(ZRootsIteratorClosure* cl) {
ZStatTimer timer(ZSubPhasePauseRootsThreads);
ResourceMark rm;
- ZRootsIteratorThreadClosure thread_cl(cl);
+ ZRootsIteratorThreadClosure thread_cl(cl, _visit_invisible);
Threads::possibly_parallel_threads_do(true, &thread_cl);
}
@@ -248,7 +255,7 @@
ZNMethod::oops_do(cl);
}
-void ZRootsIterator::oops_do(ZRootsIteratorClosure* cl, bool visit_jvmti_weak_export) {
+void ZRootsIterator::oops_do(ZRootsIteratorClosure* cl) {
ZStatTimer timer(ZSubPhasePauseRoots);
_universe.oops_do(cl);
_object_synchronizer.oops_do(cl);
@@ -259,7 +266,7 @@
if (!ClassUnloading) {
_code_cache.oops_do(cl);
}
- if (visit_jvmti_weak_export) {
+ if (_visit_jvmti_weak_export) {
_jvmti_weak_export.oops_do(cl);
}
}
--- a/src/hotspot/share/gc/z/zRootsIterator.hpp Fri Aug 23 08:48:18 2019 +0200
+++ b/src/hotspot/share/gc/z/zRootsIterator.hpp Fri Aug 23 08:48:23 2019 +0200
@@ -84,6 +84,9 @@
class ZRootsIterator {
private:
+ bool _visit_invisible;
+ bool _visit_jvmti_weak_export;
+
void do_universe(ZRootsIteratorClosure* cl);
void do_object_synchronizer(ZRootsIteratorClosure* cl);
void do_management(ZRootsIteratorClosure* cl);
@@ -103,10 +106,16 @@
ZParallelOopsDo<ZRootsIterator, &ZRootsIterator::do_code_cache> _code_cache;
public:
- ZRootsIterator();
+ ZRootsIterator(bool visit_invisible = true, bool visit_jvmti_weak_export = false);
~ZRootsIterator();
- void oops_do(ZRootsIteratorClosure* cl, bool visit_jvmti_weak_export = false);
+ void oops_do(ZRootsIteratorClosure* cl);
+};
+
+class ZRootsIteratorNoInvisible : public ZRootsIterator {
+public:
+ ZRootsIteratorNoInvisible() :
+ ZRootsIterator(false /* visit_invisible */, false /* visit_jvmti_weak_export */) {}
};
class ZConcurrentRootsIterator {
--- a/src/hotspot/share/gc/z/zThreadLocalData.hpp Fri Aug 23 08:48:18 2019 +0200
+++ b/src/hotspot/share/gc/z/zThreadLocalData.hpp Fri Aug 23 08:48:23 2019 +0200
@@ -34,10 +34,12 @@
private:
uintptr_t _address_bad_mask;
ZMarkThreadLocalStacks _stacks;
+ oop* _invisible_root;
ZThreadLocalData() :
_address_bad_mask(0),
- _stacks() {}
+ _stacks(),
+ _invisible_root(NULL) {}
static ZThreadLocalData* data(Thread* thread) {
return thread->gc_data<ZThreadLocalData>();
@@ -60,6 +62,25 @@
return &data(thread)->_stacks;
}
+ static void set_invisible_root(Thread* thread, oop* root) {
+ assert(!has_invisible_root(thread), "Already set");
+ data(thread)->_invisible_root = root;
+ }
+
+ static void clear_invisible_root(Thread* thread) {
+ assert(has_invisible_root(thread), "Should be set");
+ data(thread)->_invisible_root = NULL;
+ }
+
+ static bool has_invisible_root(Thread* thread) {
+ return data(thread)->_invisible_root != NULL;
+ }
+
+ static oop* invisible_root(Thread* thread) {
+ assert(has_invisible_root(thread), "Should be set");
+ return data(thread)->_invisible_root;
+ }
+
static ByteSize address_bad_mask_offset() {
return Thread::gc_data_offset() + byte_offset_of(ZThreadLocalData, _address_bad_mask);
}