8205923: ZGC: Verification applies load barriers before verification
authorstefank
Wed, 27 Jun 2018 15:04:27 +0200
changeset 50871 d283a214f42b
parent 50870 830b05ca7f58
child 50872 308410473abe
8205923: ZGC: Verification applies load barriers before verification Reviewed-by: pliden, eosterlund
src/hotspot/share/gc/z/zCollectedHeap.cpp
src/hotspot/share/gc/z/zHeap.cpp
src/hotspot/share/gc/z/zHeap.hpp
src/hotspot/share/gc/z/zHeapIterator.cpp
src/hotspot/share/gc/z/zHeapIterator.hpp
src/hotspot/share/gc/z/zOopClosures.cpp
src/hotspot/share/gc/z/zOopClosures.hpp
--- a/src/hotspot/share/gc/z/zCollectedHeap.cpp	Wed Jun 27 15:02:53 2018 +0200
+++ b/src/hotspot/share/gc/z/zCollectedHeap.cpp	Wed Jun 27 15:04:27 2018 +0200
@@ -233,11 +233,11 @@
 }
 
 void ZCollectedHeap::object_iterate(ObjectClosure* cl) {
-  _heap.object_iterate(cl);
+  _heap.object_iterate(cl, true /* visit_referents */);
 }
 
 void ZCollectedHeap::safe_object_iterate(ObjectClosure* cl) {
-  _heap.object_iterate(cl);
+  _heap.object_iterate(cl, true /* visit_referents */);
 }
 
 HeapWord* ZCollectedHeap::block_start(const void* addr) const {
--- a/src/hotspot/share/gc/z/zHeap.cpp	Wed Jun 27 15:02:53 2018 +0200
+++ b/src/hotspot/share/gc/z/zHeap.cpp	Wed Jun 27 15:04:27 2018 +0200
@@ -503,10 +503,10 @@
                                  used(), used_high(), used_low());
 }
 
-void ZHeap::object_iterate(ObjectClosure* cl) {
+void ZHeap::object_iterate(ObjectClosure* cl, bool visit_referents) {
   assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
 
-  ZHeapIterator iter;
+  ZHeapIterator iter(visit_referents);
   iter.objects_do(cl);
 }
 
@@ -577,6 +577,6 @@
 
   {
     ZVerifyObjectClosure cl;
-    object_iterate(&cl);
+    object_iterate(&cl, false /* visit_referents */);
   }
 }
--- a/src/hotspot/share/gc/z/zHeap.hpp	Wed Jun 27 15:02:53 2018 +0200
+++ b/src/hotspot/share/gc/z/zHeap.hpp	Wed Jun 27 15:04:27 2018 +0200
@@ -152,7 +152,7 @@
   void relocate();
 
   // Iteration
-  void object_iterate(ObjectClosure* cl);
+  void object_iterate(ObjectClosure* cl, bool visit_referents);
 
   // Serviceability
   void serviceability_initialize();
--- a/src/hotspot/share/gc/z/zHeapIterator.cpp	Wed Jun 27 15:02:53 2018 +0200
+++ b/src/hotspot/share/gc/z/zHeapIterator.cpp	Wed Jun 27 15:04:27 2018 +0200
@@ -78,14 +78,28 @@
 private:
   ZHeapIterator* const _iter;
   const oop            _base;
+  const bool           _visit_referents;
 
 public:
   ZHeapIteratorPushOopClosure(ZHeapIterator* iter, oop base) :
       _iter(iter),
-      _base(base) {}
+      _base(base),
+      _visit_referents(iter->visit_referents()) {}
+
+  oop load_oop(oop* p) {
+    if (_visit_referents) {
+      return HeapAccess<ON_UNKNOWN_OOP_REF>::oop_load_at(_base, _base->field_offset(p));
+    } else {
+      return HeapAccess<>::oop_load(p);
+    }
+  }
+
+  virtual ReferenceIterationMode reference_iteration_mode() {
+    return _visit_referents ? DO_FIELDS : DO_FIELDS_EXCEPT_REFERENT;
+  }
 
   virtual void do_oop(oop* p) {
-    const oop obj = HeapAccess<ON_UNKNOWN_OOP_REF>::oop_load_at(_base, _base->field_offset(p));
+    const oop obj = load_oop(p);
     _iter->push(obj);
   }
 
@@ -100,9 +114,10 @@
 #endif
 };
 
-ZHeapIterator::ZHeapIterator() :
+ZHeapIterator::ZHeapIterator(bool visit_referents) :
     _visit_stack(),
-    _visit_map() {}
+    _visit_map(),
+    _visit_referents(visit_referents) {}
 
 ZHeapIterator::~ZHeapIterator() {
   ZVisitMapIterator iter(&_visit_map);
@@ -163,6 +178,10 @@
   }
 }
 
+bool ZHeapIterator::visit_referents() const {
+  return _visit_referents;
+}
+
 void ZHeapIterator::objects_do(ObjectClosure* cl) {
   ZHeapIteratorRootOopClosure root_cl(this, cl);
   ZRootsIterator roots;
--- a/src/hotspot/share/gc/z/zHeapIterator.hpp	Wed Jun 27 15:02:53 2018 +0200
+++ b/src/hotspot/share/gc/z/zHeapIterator.hpp	Wed Jun 27 15:04:27 2018 +0200
@@ -42,6 +42,7 @@
 
   ZVisitStack _visit_stack;
   ZVisitMap   _visit_map;
+  const bool  _visit_referents;
 
   size_t object_index_max() const;
   size_t object_index(oop obj) const;
@@ -50,8 +51,10 @@
   void push(oop obj);
   void drain(ObjectClosure* cl);
 
+  bool visit_referents() const;
+
 public:
-  ZHeapIterator();
+  ZHeapIterator(bool visit_referents);
   ~ZHeapIterator();
 
   void objects_do(ObjectClosure* cl);
--- a/src/hotspot/share/gc/z/zOopClosures.cpp	Wed Jun 27 15:02:53 2018 +0200
+++ b/src/hotspot/share/gc/z/zOopClosures.cpp	Wed Jun 27 15:04:27 2018 +0200
@@ -40,13 +40,17 @@
             p2i(obj), p2i(p));
 }
 
-ZVerifyHeapOopClosure::ZVerifyHeapOopClosure(oop base)
-    : _base(base) {}
+OopIterateClosure::ReferenceIterationMode ZVerifyHeapOopClosure::reference_iteration_mode() {
+  // Don't visit the j.l.Reference.referents for this verification closure,
+  // since they are cleaned concurrently after ZHeap::mark_end(), and can
+  // therefore not be verified at this point.
+  return DO_FIELDS_EXCEPT_REFERENT;
+}
 
 void ZVerifyHeapOopClosure::do_oop(oop* p) {
   guarantee(ZHeap::heap()->is_in((uintptr_t)p), "oop* " PTR_FORMAT " not in heap", p2i(p));
 
-  const oop obj = HeapAccess<ON_UNKNOWN_OOP_REF>::oop_load_at(_base, _base->field_offset(p));
+  const oop obj = RawAccess<>::oop_load(p);
   z_verify_loaded_object(p, obj);
 }
 
@@ -54,10 +58,16 @@
   ShouldNotReachHere();
 }
 
+ZVerifyRootOopClosure::ZVerifyRootOopClosure() {
+  // This closure should only be used from ZHeap::mark_end(),
+  // when all roots should have been fixed by the fixup_partial_loads().
+  guarantee(ZGlobalPhase == ZPhaseMarkCompleted, "Invalid phase");
+}
+
 void ZVerifyRootOopClosure::do_oop(oop* p) {
   guarantee(!ZHeap::heap()->is_in((uintptr_t)p), "oop* " PTR_FORMAT " in heap", p2i(p));
 
-  const oop obj = NativeAccess<>::oop_load(p);
+  const oop obj = RawAccess<>::oop_load(p);
   z_verify_loaded_object(p, obj);
 }
 
@@ -66,6 +76,6 @@
 }
 
 void ZVerifyObjectClosure::do_object(oop o) {
-  ZVerifyHeapOopClosure cl(o);
+  ZVerifyHeapOopClosure cl;
   o->oop_iterate(&cl);
 }
--- a/src/hotspot/share/gc/z/zOopClosures.hpp	Wed Jun 27 15:02:53 2018 +0200
+++ b/src/hotspot/share/gc/z/zOopClosures.hpp	Wed Jun 27 15:04:27 2018 +0200
@@ -83,11 +83,8 @@
 };
 
 class ZVerifyHeapOopClosure : public BasicOopIterateClosure {
-private:
-  const oop _base;
-
 public:
-  ZVerifyHeapOopClosure(oop base);
+  virtual ReferenceIterationMode reference_iteration_mode();
 
   virtual void do_oop(oop* p);
   virtual void do_oop(narrowOop* p);
@@ -102,6 +99,8 @@
 
 class ZVerifyRootOopClosure : public OopClosure {
 public:
+  ZVerifyRootOopClosure();
+
   virtual void do_oop(oop* p);
   virtual void do_oop(narrowOop* p);
 };