# HG changeset patch
# User shade
# Date 1571690688 -7200
# Node ID 15f4471787b92dfa7e313a1591fd39b4a2fe6425
# Parent  6d5c7f91e0b590e61957d63316f58c9f34d40055
8232730: Shenandoah: Traversal should not CAS the roots
Reviewed-by: zgu, rkennke

diff -r 6d5c7f91e0b5 -r 15f4471787b9 src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp	Mon Oct 21 21:36:26 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp	Mon Oct 21 22:44:48 2019 +0200
@@ -214,15 +214,30 @@
     _mark_context(ShenandoahHeap::heap()->marking_context()) {
   }
 
-  template <class T, bool STRING_DEDUP, bool DEGEN>
+  template <class T, bool STRING_DEDUP, bool DEGEN, bool ATOMIC_UPDATE>
   void work(T* p);
 
 };
 
+class ShenandoahTraversalRootsClosure : public ShenandoahTraversalSuperClosure {
+private:
+  template <class T>
+  inline void do_oop_work(T* p)     { work<T, false, false, false>(p); }
+
+public:
+  ShenandoahTraversalRootsClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
+    ShenandoahTraversalSuperClosure(q, rp) {}
+
+  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+  virtual void do_oop(oop* p)       { do_oop_work(p); }
+
+  virtual bool do_metadata()        { return false; }
+};
+
 class ShenandoahTraversalClosure : public ShenandoahTraversalSuperClosure {
 private:
   template <class T>
-  inline void do_oop_work(T* p)     { work<T, false, false>(p); }
+  inline void do_oop_work(T* p)     { work<T, false, false, true>(p); }
 
 public:
   ShenandoahTraversalClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
@@ -237,7 +252,7 @@
 class ShenandoahTraversalMetadataClosure : public ShenandoahTraversalSuperClosure {
 private:
   template <class T>
-  inline void do_oop_work(T* p)     { work<T, false, false>(p); }
+  inline void do_oop_work(T* p)     { work<T, false, false, true>(p); }
 
 public:
   ShenandoahTraversalMetadataClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
@@ -252,7 +267,7 @@
 class ShenandoahTraversalDedupClosure : public ShenandoahTraversalSuperClosure {
 private:
   template <class T>
-  inline void do_oop_work(T* p)     { work<T, true, false>(p); }
+  inline void do_oop_work(T* p)     { work<T, true, false, true>(p); }
 
 public:
   ShenandoahTraversalDedupClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
@@ -267,7 +282,7 @@
 class ShenandoahTraversalMetadataDedupClosure : public ShenandoahTraversalSuperClosure {
 private:
   template <class T>
-  inline void do_oop_work(T* p)     { work<T, true, false>(p); }
+  inline void do_oop_work(T* p)     { work<T, true, false, true>(p); }
 
 public:
   ShenandoahTraversalMetadataDedupClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
@@ -282,7 +297,7 @@
 class ShenandoahTraversalDegenClosure : public ShenandoahTraversalSuperClosure {
 private:
   template <class T>
-  inline void do_oop_work(T* p)     { work<T, false, true>(p); }
+  inline void do_oop_work(T* p)     { work<T, false, true, false>(p); }
 
 public:
   ShenandoahTraversalDegenClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
@@ -297,7 +312,7 @@
 class ShenandoahTraversalMetadataDegenClosure : public ShenandoahTraversalSuperClosure {
 private:
   template <class T>
-  inline void do_oop_work(T* p)     { work<T, false, true>(p); }
+  inline void do_oop_work(T* p)     { work<T, false, true, false>(p); }
 
 public:
   ShenandoahTraversalMetadataDegenClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
@@ -312,7 +327,7 @@
 class ShenandoahTraversalDedupDegenClosure : public ShenandoahTraversalSuperClosure {
 private:
   template <class T>
-  inline void do_oop_work(T* p)     { work<T, true, true>(p); }
+  inline void do_oop_work(T* p)     { work<T, true, true, false>(p); }
 
 public:
   ShenandoahTraversalDedupDegenClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
@@ -327,7 +342,7 @@
 class ShenandoahTraversalMetadataDedupDegenClosure : public ShenandoahTraversalSuperClosure {
 private:
   template <class T>
-  inline void do_oop_work(T* p)     { work<T, true, true>(p); }
+  inline void do_oop_work(T* p)     { work<T, true, true, false>(p); }
 
 public:
   ShenandoahTraversalMetadataDedupDegenClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
diff -r 6d5c7f91e0b5 -r 15f4471787b9 src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp	Mon Oct 21 21:36:26 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp	Mon Oct 21 22:44:48 2019 +0200
@@ -38,9 +38,9 @@
   _heap->maybe_update_with_forwarded(p);
 }
 
-template <class T, bool STRING_DEDUP, bool DEGEN>
+template <class T, bool STRING_DEDUP, bool DEGEN, bool ATOMIC_UPDATE>
 inline void ShenandoahTraversalSuperClosure::work(T* p) {
-  _traversal_gc->process_oop<T, STRING_DEDUP, DEGEN>(p, _thread, _queue, _mark_context);
+  _traversal_gc->process_oop<T, STRING_DEDUP, DEGEN, ATOMIC_UPDATE>(p, _thread, _queue, _mark_context);
 }
 
 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHOOPCLOSURES_INLINE_HPP
diff -r 6d5c7f91e0b5 -r 15f4471787b9 src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp	Mon Oct 21 21:36:26 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp	Mon Oct 21 22:44:48 2019 +0200
@@ -186,7 +186,7 @@
 
     // Step 1: Process ordinary GC roots.
     {
-      ShenandoahTraversalClosure roots_cl(q, rp);
+      ShenandoahTraversalRootsClosure roots_cl(q, rp);
       ShenandoahMarkCLDClosure cld_cl(&roots_cl);
       MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations);
       if (unload_classes) {
@@ -266,7 +266,7 @@
     // in similar way during nmethod-register process. Therefore, we don't need to rescan code
     // roots here.
     if (!_heap->is_degenerated_gc_in_progress()) {
-      ShenandoahTraversalClosure roots_cl(q, rp);
+      ShenandoahTraversalRootsClosure roots_cl(q, rp);
       ShenandoahTraversalSATBThreadsClosure tc(&satb_cl);
       if (unload_classes) {
         ShenandoahRemarkCLDClosure remark_cld_cl(&roots_cl);
@@ -767,7 +767,7 @@
 
   template <class T>
   inline void do_oop_work(T* p) {
-    _traversal_gc->process_oop<T, false /* string dedup */, false /* degen */>(p, _thread, _queue, _mark_context);
+    _traversal_gc->process_oop<T, false /* string dedup */, false /* degen */, true /* atomic update */>(p, _thread, _queue, _mark_context);
   }
 
 public:
@@ -789,7 +789,7 @@
 
   template <class T>
   inline void do_oop_work(T* p) {
-    _traversal_gc->process_oop<T, false /* string dedup */, true /* degen */>(p, _thread, _queue, _mark_context);
+    _traversal_gc->process_oop<T, false /* string dedup */, true /* degen */, false /* atomic update */>(p, _thread, _queue, _mark_context);
   }
 
 public:
@@ -812,7 +812,7 @@
   template <class T>
   inline void do_oop_work(T* p) {
     ShenandoahEvacOOMScope evac_scope;
-    _traversal_gc->process_oop<T, false /* string dedup */, false /* degen */>(p, _thread, _queue, _mark_context);
+    _traversal_gc->process_oop<T, false /* string dedup */, false /* degen */, true /* atomic update */>(p, _thread, _queue, _mark_context);
   }
 
 public:
@@ -835,7 +835,7 @@
   template <class T>
   inline void do_oop_work(T* p) {
     ShenandoahEvacOOMScope evac_scope;
-    _traversal_gc->process_oop<T, false /* string dedup */, true /* degen */>(p, _thread, _queue, _mark_context);
+    _traversal_gc->process_oop<T, false /* string dedup */, true /* degen */, false /* atomic update */>(p, _thread, _queue, _mark_context);
   }
 
 public:
diff -r 6d5c7f91e0b5 -r 15f4471787b9 src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.hpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.hpp	Mon Oct 21 21:36:26 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.hpp	Mon Oct 21 22:44:48 2019 +0200
@@ -48,7 +48,7 @@
   void concurrent_traversal_collection();
   void final_traversal_collection();
 
-  template <class T, bool STRING_DEDUP, bool DEGEN>
+  template <class T, bool STRING_DEDUP, bool DEGEN, bool ATOMIC_UPDATE>
   inline void process_oop(T* p, Thread* thread, ShenandoahObjToScanQueue* queue, ShenandoahMarkingContext* const mark_context);
 
   bool check_and_handle_cancelled_gc(ShenandoahTaskTerminator* terminator, bool sts_yield);
diff -r 6d5c7f91e0b5 -r 15f4471787b9 src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.inline.hpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.inline.hpp	Mon Oct 21 21:36:26 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.inline.hpp	Mon Oct 21 22:44:48 2019 +0200
@@ -35,12 +35,13 @@
 #include "oops/compressedOops.inline.hpp"
 #include "oops/oop.inline.hpp"
 
-template <class T, bool STRING_DEDUP, bool DEGEN>
+template <class T, bool STRING_DEDUP, bool DEGEN, bool ATOMIC_UPDATE>
 void ShenandoahTraversalGC::process_oop(T* p, Thread* thread, ShenandoahObjToScanQueue* queue, ShenandoahMarkingContext* const mark_context) {
   T o = RawAccess<>::oop_load(p);
   if (!CompressedOops::is_null(o)) {
     oop obj = CompressedOops::decode_not_null(o);
     if (DEGEN) {
+      assert(!ATOMIC_UPDATE, "Degen path assumes non-atomic updates");
       oop forw = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
       if (obj != forw) {
         // Update reference.
@@ -54,7 +55,11 @@
       }
       shenandoah_assert_forwarded_except(p, obj, _heap->cancelled_gc());
       // Update reference.
-      ShenandoahHeap::cas_oop(forw, p, obj);
+      if (ATOMIC_UPDATE) {
+        ShenandoahHeap::cas_oop(forw, p, obj);
+      } else {
+        RawAccess<IS_NOT_NULL>::oop_store(p, forw);
+      }
       obj = forw;
     }