# HG changeset patch # User stefank # Date 1560164652 -7200 # Node ID 3868dde58ebb411f99419d5c9f830977730eace6 # Parent 73395f9cad54116f33b1ed9e725ffb4150a94aa5 8227175: ZGC: ZHeapIterator visits potentially dead objects Reviewed-by: zgu, eosterlund diff -r 73395f9cad54 -r 3868dde58ebb src/hotspot/share/classfile/classLoaderData.cpp --- a/src/hotspot/share/classfile/classLoaderData.cpp Tue Jul 02 12:24:26 2019 +0200 +++ b/src/hotspot/share/classfile/classLoaderData.cpp Mon Jun 10 13:04:12 2019 +0200 @@ -266,6 +266,19 @@ } #endif // PRODUCT +void ClassLoaderData::clear_claim(int claim) { + for (;;) { + int old_claim = Atomic::load(&_claim); + if ((old_claim & claim) == 0) { + return; + } + int new_claim = old_claim & ~claim; + if (Atomic::cmpxchg(new_claim, &_claim, old_claim) == old_claim) { + return; + } + } +} + bool ClassLoaderData::try_claim(int claim) { for (;;) { int old_claim = Atomic::load(&_claim); diff -r 73395f9cad54 -r 3868dde58ebb src/hotspot/share/classfile/classLoaderData.hpp --- a/src/hotspot/share/classfile/classLoaderData.hpp Tue Jul 02 12:24:26 2019 +0200 +++ b/src/hotspot/share/classfile/classLoaderData.hpp Mon Jun 10 13:04:12 2019 +0200 @@ -206,16 +206,17 @@ // The "claim" is typically used to check if oops_do needs to be applied on // the CLD or not. Most GCs only perform strong marking during the marking phase. - enum { - _claim_none = 0, - _claim_finalizable = 2, - _claim_strong = 3 + enum Claim { + _claim_none = 0, + _claim_finalizable = 2, + _claim_strong = 3, + _claim_other = 4 }; void clear_claim() { _claim = 0; } + void clear_claim(int claim); bool claimed() const { return _claim != 0; } + bool claimed(int claim) const { return (_claim & claim) == claim; } bool try_claim(int claim); - int get_claim() const { return _claim; } - void set_claim(int claim) { _claim = claim; } // Computes if the CLD is alive or not. This is safe to call in concurrent // contexts. diff -r 73395f9cad54 -r 3868dde58ebb src/hotspot/share/classfile/classLoaderDataGraph.cpp --- a/src/hotspot/share/classfile/classLoaderDataGraph.cpp Tue Jul 02 12:24:26 2019 +0200 +++ b/src/hotspot/share/classfile/classLoaderDataGraph.cpp Mon Jun 10 13:04:12 2019 +0200 @@ -64,6 +64,11 @@ } } +void ClassLoaderDataGraph::clear_claimed_marks(int claim) { + for (ClassLoaderData* cld = OrderAccess::load_acquire(&_head); cld != NULL; cld = cld->next()) { + cld->clear_claim(claim); + } +} // Class iterator used by the compiler. It gets some number of classes at // a safepoint to decay invocation counters on the methods. class ClassLoaderDataGraphKlassIteratorStatic { @@ -471,7 +476,7 @@ // The CLDs in [_head, _saved_head] were all added during last call to remember_new_clds(true); ClassLoaderData* curr = _head; while (curr != _saved_head) { - if (!curr->claimed()) { + if (!curr->claimed(ClassLoaderData::_claim_strong)) { array->push(curr); LogTarget(Debug, class, loader, data) lt; if (lt.is_enabled()) { diff -r 73395f9cad54 -r 3868dde58ebb src/hotspot/share/classfile/classLoaderDataGraph.hpp --- a/src/hotspot/share/classfile/classLoaderDataGraph.hpp Tue Jul 02 12:24:26 2019 +0200 +++ b/src/hotspot/share/classfile/classLoaderDataGraph.hpp Mon Jun 10 13:04:12 2019 +0200 @@ -68,6 +68,7 @@ static void clean_module_and_package_info(); static void purge(); static void clear_claimed_marks(); + static void clear_claimed_marks(int claim); // Iteration through CLDG inside a safepoint; GC support static void cld_do(CLDClosure* cl); static void cld_unloading_do(CLDClosure* cl); diff -r 73395f9cad54 -r 3868dde58ebb src/hotspot/share/gc/z/zArguments.cpp --- a/src/hotspot/share/gc/z/zArguments.cpp Tue Jul 02 12:24:26 2019 +0200 +++ b/src/hotspot/share/gc/z/zArguments.cpp Mon Jun 10 13:04:12 2019 +0200 @@ -92,6 +92,11 @@ // same reason we need fixup_partial_loads FLAG_SET_DEFAULT(VerifyBeforeIteration, false); + if (VerifyBeforeGC || VerifyDuringGC || VerifyAfterGC) { + FLAG_SET_DEFAULT(ZVerifyRoots, true); + FLAG_SET_DEFAULT(ZVerifyObjects, true); + } + // Verification of stacks not (yet) supported, for the same reason // we need fixup_partial_loads DEBUG_ONLY(FLAG_SET_DEFAULT(VerifyStack, false)); diff -r 73395f9cad54 -r 3868dde58ebb src/hotspot/share/gc/z/zCollectedHeap.cpp --- a/src/hotspot/share/gc/z/zCollectedHeap.cpp Tue Jul 02 12:24:26 2019 +0200 +++ b/src/hotspot/share/gc/z/zCollectedHeap.cpp Mon Jun 10 13:04:12 2019 +0200 @@ -232,11 +232,11 @@ } void ZCollectedHeap::object_iterate(ObjectClosure* cl) { - _heap.object_iterate(cl, true /* visit_referents */); + _heap.object_iterate(cl, true /* visit_weaks */); } void ZCollectedHeap::safe_object_iterate(ObjectClosure* cl) { - _heap.object_iterate(cl, true /* visit_referents */); + _heap.object_iterate(cl, true /* visit_weaks */); } HeapWord* ZCollectedHeap::block_start(const void* addr) const { diff -r 73395f9cad54 -r 3868dde58ebb src/hotspot/share/gc/z/zDriver.cpp --- a/src/hotspot/share/gc/z/zDriver.cpp Tue Jul 02 12:24:26 2019 +0200 +++ b/src/hotspot/share/gc/z/zDriver.cpp Mon Jun 10 13:04:12 2019 +0200 @@ -31,6 +31,7 @@ #include "gc/z/zMessagePort.inline.hpp" #include "gc/z/zServiceability.hpp" #include "gc/z/zStat.hpp" +#include "gc/z/zVerify.hpp" #include "logging/log.hpp" #include "memory/universe.hpp" #include "runtime/vmOperations.hpp" @@ -86,6 +87,9 @@ GCIdMark gc_id_mark(_gc_id); IsGCActiveMark gc_active_mark; + // Verify roots + ZVerify::roots_strong(); + // Execute operation _success = do_operation(); @@ -301,8 +305,14 @@ void ZDriver::pause_verify() { if (VerifyBeforeGC || VerifyDuringGC || VerifyAfterGC) { + // Full verification VM_Verify op; VMThread::execute(&op); + + } else if (ZVerifyRoots || ZVerifyObjects) { + // Limited verification + VM_ZVerifyOperation op; + VMThread::execute(&op); } } diff -r 73395f9cad54 -r 3868dde58ebb src/hotspot/share/gc/z/zHeap.cpp --- a/src/hotspot/share/gc/z/zHeap.cpp Tue Jul 02 12:24:26 2019 +0200 +++ b/src/hotspot/share/gc/z/zHeap.cpp Mon Jun 10 13:04:12 2019 +0200 @@ -41,6 +41,7 @@ #include "gc/z/zTask.hpp" #include "gc/z/zThread.hpp" #include "gc/z/zTracer.inline.hpp" +#include "gc/z/zVerify.hpp" #include "gc/z/zVirtualMemory.inline.hpp" #include "gc/z/zWorkers.inline.hpp" #include "logging/log.hpp" @@ -340,6 +341,9 @@ // Enter mark completed phase ZGlobalPhase = ZPhaseMarkCompleted; + // Verify after mark + ZVerify::after_mark(); + // Update statistics ZStatSample(ZSamplerHeapUsedAfterMark, used()); ZStatHeap::set_at_mark_end(capacity(), allocated(), used()); @@ -468,11 +472,11 @@ used(), used_high(), used_low()); } -void ZHeap::object_iterate(ObjectClosure* cl, bool visit_referents) { +void ZHeap::object_iterate(ObjectClosure* cl, bool visit_weaks) { assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); ZHeapIterator iter; - iter.objects_do(cl, visit_referents); + iter.objects_do(cl, visit_weaks); } void ZHeap::serviceability_initialize() { @@ -518,40 +522,11 @@ st->cr(); } -class ZVerifyRootsTask : public ZTask { -private: - ZStatTimerDisable _disable; - ZRootsIterator _strong_roots; - ZWeakRootsIterator _weak_roots; - -public: - ZVerifyRootsTask() : - ZTask("ZVerifyRootsTask"), - _disable(), - _strong_roots(), - _weak_roots() {} - - virtual void work() { - ZStatTimerDisable disable; - ZVerifyOopClosure cl; - _strong_roots.oops_do(&cl); - _weak_roots.oops_do(&cl); - } -}; - void ZHeap::verify() { // Heap verification can only be done between mark end and // relocate start. This is the only window where all oop are // good and the whole heap is in a consistent state. guarantee(ZGlobalPhase == ZPhaseMarkCompleted, "Invalid phase"); - { - ZVerifyRootsTask task; - _workers.run_parallel(&task); - } - - { - ZVerifyObjectClosure cl; - object_iterate(&cl, false /* visit_referents */); - } + ZVerify::after_weak_processing(); } diff -r 73395f9cad54 -r 3868dde58ebb src/hotspot/share/gc/z/zHeap.hpp --- a/src/hotspot/share/gc/z/zHeap.hpp Tue Jul 02 12:24:26 2019 +0200 +++ b/src/hotspot/share/gc/z/zHeap.hpp Mon Jun 10 13:04:12 2019 +0200 @@ -161,7 +161,7 @@ void relocate(); // Iteration - void object_iterate(ObjectClosure* cl, bool visit_referents); + void object_iterate(ObjectClosure* cl, bool visit_weaks); // Serviceability void serviceability_initialize(); diff -r 73395f9cad54 -r 3868dde58ebb src/hotspot/share/gc/z/zHeapIterator.cpp --- a/src/hotspot/share/gc/z/zHeapIterator.cpp Tue Jul 02 12:24:26 2019 +0200 +++ b/src/hotspot/share/gc/z/zHeapIterator.cpp Mon Jun 10 13:04:12 2019 +0200 @@ -22,6 +22,8 @@ */ #include "precompiled.hpp" +#include "classfile/classLoaderData.hpp" +#include "classfile/classLoaderDataGraph.hpp" #include "gc/z/zBarrier.inline.hpp" #include "gc/z/zGlobals.hpp" #include "gc/z/zGranuleMap.inline.hpp" @@ -83,7 +85,7 @@ }; template -class ZHeapIteratorOopClosure : public BasicOopIterateClosure { +class ZHeapIteratorOopClosure : public ClaimMetadataVisitingOopIterateClosure { private: ZHeapIterator* const _iter; const oop _base; @@ -98,6 +100,7 @@ public: ZHeapIteratorOopClosure(ZHeapIterator* iter, oop base) : + ClaimMetadataVisitingOopIterateClosure(ClassLoaderData::_claim_other), _iter(iter), _base(base) {} @@ -130,6 +133,7 @@ for (ZHeapIteratorBitMap* map; iter.next(&map);) { delete map; } + ClassLoaderDataGraph::clear_claimed_marks(ClassLoaderData::_claim_other); } static size_t object_index_max() { @@ -184,15 +188,23 @@ obj->oop_iterate(&cl); } -template +class ZHeapIterateConcurrentRootsIterator : public ZConcurrentRootsIterator { +public: + ZHeapIterateConcurrentRootsIterator() : + ZConcurrentRootsIterator(ClassLoaderData::_claim_other) {} +}; + +template void ZHeapIterator::objects_do(ObjectClosure* cl) { ZStatTimerDisable disable; // Push roots to visit - push_roots(); - push_roots(); - push_roots(); - push_roots(); + push_roots(); + push_roots(); + if (VisitWeaks) { + push_roots(); + push_roots(); + } // Drain stack while (!_visit_stack.is_empty()) { @@ -202,14 +214,14 @@ cl->do_object(obj); // Push fields to visit - push_fields(obj); + push_fields(obj); } } -void ZHeapIterator::objects_do(ObjectClosure* cl, bool visit_referents) { - if (visit_referents) { - objects_do(cl); +void ZHeapIterator::objects_do(ObjectClosure* cl, bool visit_weaks) { + if (visit_weaks) { + objects_do(cl); } else { - objects_do(cl); + objects_do(cl); } } diff -r 73395f9cad54 -r 3868dde58ebb src/hotspot/share/gc/z/zHeapIterator.hpp --- a/src/hotspot/share/gc/z/zHeapIterator.hpp Tue Jul 02 12:24:26 2019 +0200 +++ b/src/hotspot/share/gc/z/zHeapIterator.hpp Mon Jun 10 13:04:12 2019 +0200 @@ -54,7 +54,7 @@ ZHeapIterator(); ~ZHeapIterator(); - void objects_do(ObjectClosure* cl, bool visit_referents); + void objects_do(ObjectClosure* cl, bool visit_weaks); }; #endif // SHARE_GC_Z_ZHEAPITERATOR_HPP diff -r 73395f9cad54 -r 3868dde58ebb src/hotspot/share/gc/z/zMark.cpp --- a/src/hotspot/share/gc/z/zMark.cpp Tue Jul 02 12:24:26 2019 +0200 +++ b/src/hotspot/share/gc/z/zMark.cpp Mon Jun 10 13:04:12 2019 +0200 @@ -22,6 +22,7 @@ */ #include "precompiled.hpp" +#include "classfile/classLoaderDataGraph.hpp" #include "gc/z/zBarrier.inline.hpp" #include "gc/z/zMark.inline.hpp" #include "gc/z/zMarkCache.inline.hpp" @@ -632,14 +633,23 @@ class ZMarkConcurrentRootsTask : public ZTask { private: + SuspendibleThreadSetJoiner _sts_joiner; ZConcurrentRootsIterator _roots; ZMarkConcurrentRootsIteratorClosure _cl; public: ZMarkConcurrentRootsTask(ZMark* mark) : ZTask("ZMarkConcurrentRootsTask"), - _roots(true /* marking */), - _cl() {} + _sts_joiner(true /* active */), + _roots(ClassLoaderData::_claim_strong), + _cl() { + ClassLoaderDataGraph_lock->lock(); + ClassLoaderDataGraph::clear_claimed_marks(); + } + + ~ZMarkConcurrentRootsTask() { + ClassLoaderDataGraph_lock->unlock(); + } virtual void work() { _roots.oops_do(&_cl); diff -r 73395f9cad54 -r 3868dde58ebb src/hotspot/share/gc/z/zOopClosures.cpp --- a/src/hotspot/share/gc/z/zOopClosures.cpp Tue Jul 02 12:24:26 2019 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -#include "precompiled.hpp" -#include "gc/z/zHeap.hpp" -#include "gc/z/zOopClosures.inline.hpp" -#include "gc/z/zOop.inline.hpp" -#include "memory/iterator.inline.hpp" -#include "oops/access.inline.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/safepoint.hpp" -#include "utilities/debug.hpp" -#include "utilities/globalDefinitions.hpp" - -void ZVerifyOopClosure::do_oop(oop* p) { - guarantee(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint"); - guarantee(ZGlobalPhase == ZPhaseMarkCompleted, "Invalid phase"); - guarantee(!ZResurrection::is_blocked(), "Invalid phase"); - - const oop o = RawAccess<>::oop_load(p); - if (o != NULL) { - const uintptr_t addr = ZOop::to_address(o); - const uintptr_t good_addr = ZAddress::good(addr); - guarantee(ZAddress::is_good(addr) || ZAddress::is_finalizable_good(addr), - "Bad oop " PTR_FORMAT " found at " PTR_FORMAT ", expected " PTR_FORMAT, - addr, p2i(p), good_addr); - guarantee(oopDesc::is_oop(ZOop::from_address(good_addr)), - "Bad object " PTR_FORMAT " found at " PTR_FORMAT, - addr, p2i(p)); - } -} - -void ZVerifyOopClosure::do_oop(narrowOop* p) { - ShouldNotReachHere(); -} - -void ZVerifyObjectClosure::do_object(oop o) { - ZVerifyOopClosure cl; - o->oop_iterate(&cl); -} diff -r 73395f9cad54 -r 3868dde58ebb src/hotspot/share/gc/z/zOopClosures.hpp --- a/src/hotspot/share/gc/z/zOopClosures.hpp Tue Jul 02 12:24:26 2019 +0200 +++ b/src/hotspot/share/gc/z/zOopClosures.hpp Mon Jun 10 13:04:12 2019 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -46,16 +46,13 @@ }; template -class ZMarkBarrierOopClosure : public MetadataVisitingOopIterateClosure { +class ZMarkBarrierOopClosure : public ClaimMetadataVisitingOopIterateClosure { public: ZMarkBarrierOopClosure(); virtual void do_oop(oop* p); virtual void do_oop(narrowOop* p); - virtual void do_klass(Klass* k); - virtual void do_cld(ClassLoaderData* cld); - #ifdef ASSERT virtual bool should_verify_oops() { return false; @@ -80,26 +77,4 @@ virtual void do_oop(narrowOop* p); }; -class ZVerifyOopClosure : public ZRootsIteratorClosure, public BasicOopIterateClosure { -public: - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); - - virtual ReferenceIterationMode reference_iteration_mode() { - return DO_FIELDS; - } - -#ifdef ASSERT - // Verification handled by the closure itself - virtual bool should_verify_oops() { - return false; - } -#endif -}; - -class ZVerifyObjectClosure : public ObjectClosure { -public: - virtual void do_object(oop o); -}; - #endif // SHARE_GC_Z_ZOOPCLOSURES_HPP diff -r 73395f9cad54 -r 3868dde58ebb src/hotspot/share/gc/z/zOopClosures.inline.hpp --- a/src/hotspot/share/gc/z/zOopClosures.inline.hpp Tue Jul 02 12:24:26 2019 +0200 +++ b/src/hotspot/share/gc/z/zOopClosures.inline.hpp Mon Jun 10 13:04:12 2019 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -55,7 +55,12 @@ template inline ZMarkBarrierOopClosure::ZMarkBarrierOopClosure() : - MetadataVisitingOopIterateClosure(finalizable ? NULL : ZHeap::heap()->reference_discoverer()) {} + ClaimMetadataVisitingOopIterateClosure(finalizable + ? ClassLoaderData::_claim_finalizable + : ClassLoaderData::_claim_strong, + finalizable + ? NULL + : ZHeap::heap()->reference_discoverer()) {} template inline void ZMarkBarrierOopClosure::do_oop(oop* p) { @@ -67,18 +72,6 @@ ShouldNotReachHere(); } -template -inline void ZMarkBarrierOopClosure::do_klass(Klass* k) { - ClassLoaderData* const cld = k->class_loader_data(); - ZMarkBarrierOopClosure::do_cld(cld); -} - -template -inline void ZMarkBarrierOopClosure::do_cld(ClassLoaderData* cld) { - const int claim = finalizable ? ClassLoaderData::_claim_finalizable : ClassLoaderData::_claim_strong; - cld->oops_do(this, claim); -} - inline bool ZPhantomIsAliveObjectClosure::do_object_b(oop o) { return ZBarrier::is_alive_barrier_on_phantom_oop(o); } diff -r 73395f9cad54 -r 3868dde58ebb src/hotspot/share/gc/z/zRootsIterator.cpp --- a/src/hotspot/share/gc/z/zRootsIterator.cpp Tue Jul 02 12:24:26 2019 +0200 +++ b/src/hotspot/share/gc/z/zRootsIterator.cpp Mon Jun 10 13:04:12 2019 +0200 @@ -263,24 +263,16 @@ } } -ZConcurrentRootsIterator::ZConcurrentRootsIterator(bool marking) : - _marking(marking), - _sts_joiner(marking /* active */), +ZConcurrentRootsIterator::ZConcurrentRootsIterator(int cld_claim) : _jni_handles_iter(JNIHandles::global_handles()), + _cld_claim(cld_claim), _jni_handles(this), _class_loader_data_graph(this) { ZStatTimer timer(ZSubPhaseConcurrentRootsSetup); - if (_marking) { - ClassLoaderDataGraph_lock->lock(); - ClassLoaderDataGraph::clear_claimed_marks(); - } } ZConcurrentRootsIterator::~ZConcurrentRootsIterator() { ZStatTimer timer(ZSubPhaseConcurrentRootsTeardown); - if (_marking) { - ClassLoaderDataGraph_lock->unlock(); - } } void ZConcurrentRootsIterator::do_jni_handles(ZRootsIteratorClosure* cl) { @@ -290,13 +282,8 @@ void ZConcurrentRootsIterator::do_class_loader_data_graph(ZRootsIteratorClosure* cl) { ZStatTimer timer(ZSubPhaseConcurrentRootsClassLoaderDataGraph); - if (_marking) { - CLDToOopClosure cld_cl(cl, ClassLoaderData::_claim_strong); - ClassLoaderDataGraph::always_strong_cld_do(&cld_cl); - } else { - CLDToOopClosure cld_cl(cl, ClassLoaderData::_claim_none); - ClassLoaderDataGraph::cld_do(&cld_cl); - } + CLDToOopClosure cld_cl(cl, _cld_claim); + ClassLoaderDataGraph::always_strong_cld_do(&cld_cl); } void ZConcurrentRootsIterator::oops_do(ZRootsIteratorClosure* cl) { diff -r 73395f9cad54 -r 3868dde58ebb src/hotspot/share/gc/z/zRootsIterator.hpp --- a/src/hotspot/share/gc/z/zRootsIterator.hpp Tue Jul 02 12:24:26 2019 +0200 +++ b/src/hotspot/share/gc/z/zRootsIterator.hpp Mon Jun 10 13:04:12 2019 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -111,9 +111,8 @@ class ZConcurrentRootsIterator { private: - const bool _marking; - SuspendibleThreadSetJoiner _sts_joiner; ZOopStorageIterator _jni_handles_iter; + int _cld_claim; void do_jni_handles(ZRootsIteratorClosure* cl); void do_class_loader_data_graph(ZRootsIteratorClosure* cl); @@ -122,7 +121,7 @@ ZParallelOopsDo _class_loader_data_graph; public: - ZConcurrentRootsIterator(bool marking = false); + ZConcurrentRootsIterator(int cld_claim); ~ZConcurrentRootsIterator(); void oops_do(ZRootsIteratorClosure* cl); diff -r 73395f9cad54 -r 3868dde58ebb src/hotspot/share/gc/z/zVerify.cpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/hotspot/share/gc/z/zVerify.cpp Mon Jun 10 13:04:12 2019 +0200 @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "classfile/classLoaderData.hpp" +#include "classfile/classLoaderDataGraph.hpp" +#include "gc/z/zAddress.hpp" +#include "gc/z/zHeap.inline.hpp" +#include "gc/z/zOop.hpp" +#include "gc/z/zResurrection.hpp" +#include "gc/z/zRootsIterator.hpp" +#include "gc/z/zStat.hpp" +#include "gc/z/zVerify.hpp" +#include "memory/allocation.hpp" +#include "memory/iterator.inline.hpp" +#include "oops/oop.inline.hpp" + +#define BAD_OOP_REPORT(addr) \ + "Bad oop " PTR_FORMAT " found at " PTR_FORMAT ", expected " PTR_FORMAT, \ + addr, p2i(p), ZAddress::good(addr) + +class ZVerifyRootsClosure : public ZRootsIteratorClosure { +public: + virtual void do_oop(oop* p) { + uintptr_t value = ZOop::to_address(*p); + + if (value == 0) { + return; + } + + guarantee(!ZAddress::is_finalizable(value), BAD_OOP_REPORT(value)); + guarantee(ZAddress::is_good(value), BAD_OOP_REPORT(value)); + guarantee(oopDesc::is_oop(ZOop::from_address(value)), BAD_OOP_REPORT(value)); + } + virtual void do_oop(narrowOop*) { ShouldNotReachHere(); } +}; + +template +class ZVerifyOopClosure : public ClaimMetadataVisitingOopIterateClosure, public ZRootsIteratorClosure { +public: + ZVerifyOopClosure() : + ClaimMetadataVisitingOopIterateClosure(ClassLoaderData::_claim_other) {} + + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p) { ShouldNotReachHere(); } + + virtual ReferenceIterationMode reference_iteration_mode() { + return VisitReferents ? DO_FIELDS : DO_FIELDS_EXCEPT_REFERENT; + } + +#ifdef ASSERT + // Verification handled by the closure itself + virtual bool should_verify_oops() { + return false; + } +#endif +}; + +class ZVerifyObjectClosure : public ObjectClosure { +private: + bool _visit_referents; + +public: + ZVerifyObjectClosure(bool visit_referents) : _visit_referents(visit_referents) {} + virtual void do_object(oop o); +}; + +template +void ZVerify::roots_impl() { + if (ZVerifyRoots) { + ZVerifyRootsClosure cl; + RootsIterator iter; + iter.oops_do(&cl); + } +} + +void ZVerify::roots_strong() { + roots_impl(); +} + +class ZVerifyConcurrentRootsIterator : public ZConcurrentRootsIterator { +public: + ZVerifyConcurrentRootsIterator() + : ZConcurrentRootsIterator(ClassLoaderData::_claim_none) {} +}; + +void ZVerify::roots_concurrent() { + roots_impl(); +} + +void ZVerify::roots_weak() { + assert(!ZResurrection::is_blocked(), "Invalid phase"); + + roots_impl(); +} + +void ZVerify::roots(bool verify_weaks) { + roots_strong(); + roots_concurrent(); + if (verify_weaks) { + roots_weak(); + roots_concurrent_weak(); + } +} + +void ZVerify::objects(bool verify_weaks) { + if (ZVerifyObjects) { + ZVerifyObjectClosure cl(verify_weaks); + ZHeap::heap()->object_iterate(&cl, verify_weaks); + } +} + +void ZVerify::roots_concurrent_weak() { + assert(!ZResurrection::is_blocked(), "Invalid phase"); + + roots_impl(); +} + +void ZVerify::roots_and_objects(bool verify_weaks) { + ZStatTimerDisable _disable; + + roots(verify_weaks); + objects(verify_weaks); +} + +void ZVerify::after_mark() { + // Only verify strong roots and references. + roots_and_objects(false /* verify_weaks */); +} + +void ZVerify::after_weak_processing() { + // Also verify weaks - all should have been processed at this point. + roots_and_objects(true /* verify_weaks */); +} + +template +void ZVerifyOopClosure::do_oop(oop* p) { + guarantee(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint"); + guarantee(ZGlobalPhase == ZPhaseMarkCompleted, "Invalid phase"); + guarantee(!ZResurrection::is_blocked(), "Invalid phase"); + + const oop o = RawAccess<>::oop_load(p); + if (o == NULL) { + return; + } + + const uintptr_t addr = ZOop::to_address(o); + if (VisitReferents) { + guarantee(ZAddress::is_good(addr) || ZAddress::is_finalizable_good(addr), BAD_OOP_REPORT(addr)); + } else { + // Should not encounter finalizable oops through strong-only paths. Assumes only strong roots are visited. + guarantee(ZAddress::is_good(addr), BAD_OOP_REPORT(addr)); + } + + const uintptr_t good_addr = ZAddress::good(addr); + guarantee(oopDesc::is_oop(ZOop::from_address(good_addr)), BAD_OOP_REPORT(addr)); +} + +void ZVerifyObjectClosure::do_object(oop o) { + if (_visit_referents) { + ZVerifyOopClosure cl; + o->oop_iterate((OopIterateClosure*)&cl); + } else { + ZVerifyOopClosure cl; + o->oop_iterate(&cl); + } +} diff -r 73395f9cad54 -r 3868dde58ebb src/hotspot/share/gc/z/zVerify.hpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/hotspot/share/gc/z/zVerify.hpp Mon Jun 10 13:04:12 2019 +0200 @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZVERIFY_HPP +#define SHARE_GC_Z_ZVERIFY_HPP + +#include "memory/allocation.hpp" + +class ZVerify : public AllStatic { +private: + template + static void roots_impl(); + static void roots(bool verify_weaks); + + static void roots_weak(); + static void roots_concurrent(); + static void roots_concurrent_weak(); + + static void objects(bool verify_weaks); + + static void roots_and_objects(bool visit_weaks); + +public: + // Verify strong (non-concurrent) roots. Should always be good. + static void roots_strong(); + + // Verify all strong roots and references after marking. + static void after_mark(); + + // Verify strong and weak roots and references. + static void after_weak_processing(); +}; + +class VM_ZVerifyOperation : public VM_Operation { +public: + virtual bool needs_inactive_gc_locker() const { + // An inactive GC locker is needed in operations where we change the bad + // mask or move objects. Changing the bad mask will invalidate all oops, + // which makes it conceptually the same thing as moving all objects. + return false; + } + + virtual void doit() { + ZVerify::after_weak_processing(); + } + + bool success() const { + return true; + } + + virtual VMOp_Type type() const { return VMOp_ZVerify; } +}; + +#endif // SHARE_GC_Z_ZVERIFY_HPP diff -r 73395f9cad54 -r 3868dde58ebb src/hotspot/share/gc/z/z_globals.hpp --- a/src/hotspot/share/gc/z/z_globals.hpp Tue Jul 02 12:24:26 2019 +0200 +++ b/src/hotspot/share/gc/z/z_globals.hpp Mon Jun 10 13:04:12 2019 +0200 @@ -76,6 +76,12 @@ diagnostic(bool, ZVerifyViews, false, \ "Verify heap view accesses") \ \ + diagnostic(bool, ZVerifyRoots, trueInDebug, \ + "Verify roots") \ + \ + diagnostic(bool, ZVerifyObjects, false, \ + "Verify objects") \ + \ diagnostic(bool, ZVerifyMarking, false, \ "Verify marking stacks") \ \ diff -r 73395f9cad54 -r 3868dde58ebb src/hotspot/share/memory/iterator.hpp --- a/src/hotspot/share/memory/iterator.hpp Tue Jul 02 12:24:26 2019 +0200 +++ b/src/hotspot/share/memory/iterator.hpp Mon Jun 10 13:04:12 2019 +0200 @@ -144,18 +144,28 @@ void do_cld(ClassLoaderData* cld); }; -// The base class for all concurrent marking closures, -// that participates in class unloading. -// It's used to proxy through the metadata to the oops defined in them. -class MetadataVisitingOopIterateClosure: public OopIterateClosure { +class ClaimMetadataVisitingOopIterateClosure : public OopIterateClosure { + protected: + const int _claim; + public: - MetadataVisitingOopIterateClosure(ReferenceDiscoverer* rd = NULL) : OopIterateClosure(rd) { } + ClaimMetadataVisitingOopIterateClosure(int claim, ReferenceDiscoverer* rd = NULL) : + OopIterateClosure(rd), + _claim(claim) { } virtual bool do_metadata() { return true; } virtual void do_klass(Klass* k); virtual void do_cld(ClassLoaderData* cld); }; +// The base class for all concurrent marking closures, +// that participates in class unloading. +// It's used to proxy through the metadata to the oops defined in them. +class MetadataVisitingOopIterateClosure: public ClaimMetadataVisitingOopIterateClosure { + public: + MetadataVisitingOopIterateClosure(ReferenceDiscoverer* rd = NULL); +}; + // ObjectClosure is used for iterating through an object space class ObjectClosure : public Closure { diff -r 73395f9cad54 -r 3868dde58ebb src/hotspot/share/memory/iterator.inline.hpp --- a/src/hotspot/share/memory/iterator.inline.hpp Tue Jul 02 12:24:26 2019 +0200 +++ b/src/hotspot/share/memory/iterator.inline.hpp Mon Jun 10 13:04:12 2019 +0200 @@ -39,13 +39,17 @@ #include "oops/typeArrayKlass.inline.hpp" #include "utilities/debug.hpp" -inline void MetadataVisitingOopIterateClosure::do_cld(ClassLoaderData* cld) { - cld->oops_do(this, ClassLoaderData::_claim_strong); +// Defaults to strong claiming. +inline MetadataVisitingOopIterateClosure::MetadataVisitingOopIterateClosure(ReferenceDiscoverer* rd) : + ClaimMetadataVisitingOopIterateClosure(ClassLoaderData::_claim_strong, rd) {} + +inline void ClaimMetadataVisitingOopIterateClosure::do_cld(ClassLoaderData* cld) { + cld->oops_do(this, _claim); } -inline void MetadataVisitingOopIterateClosure::do_klass(Klass* k) { +inline void ClaimMetadataVisitingOopIterateClosure::do_klass(Klass* k) { ClassLoaderData* cld = k->class_loader_data(); - MetadataVisitingOopIterateClosure::do_cld(cld); + ClaimMetadataVisitingOopIterateClosure::do_cld(cld); } #ifdef ASSERT diff -r 73395f9cad54 -r 3868dde58ebb src/hotspot/share/runtime/vmOperations.hpp --- a/src/hotspot/share/runtime/vmOperations.hpp Tue Jul 02 12:24:26 2019 +0200 +++ b/src/hotspot/share/runtime/vmOperations.hpp Mon Jun 10 13:04:12 2019 +0200 @@ -72,6 +72,7 @@ template(ZMarkStart) \ template(ZMarkEnd) \ template(ZRelocateStart) \ + template(ZVerify) \ template(HandshakeOneThread) \ template(HandshakeAllThreads) \ template(HandshakeFallback) \