8232730: Shenandoah: Traversal should not CAS the roots
Reviewed-by: zgu, rkennke
--- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp Mon Oct 21 21:36:26 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp Mon Oct 21 22:44:48 2019 +0200
@@ -214,15 +214,30 @@
_mark_context(ShenandoahHeap::heap()->marking_context()) {
}
- template <class T, bool STRING_DEDUP, bool DEGEN>
+ template <class T, bool STRING_DEDUP, bool DEGEN, bool ATOMIC_UPDATE>
void work(T* p);
};
+class ShenandoahTraversalRootsClosure : public ShenandoahTraversalSuperClosure {
+private:
+ template <class T>
+ inline void do_oop_work(T* p) { work<T, false, false, false>(p); }
+
+public:
+ ShenandoahTraversalRootsClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
+ ShenandoahTraversalSuperClosure(q, rp) {}
+
+ virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+ virtual void do_oop(oop* p) { do_oop_work(p); }
+
+ virtual bool do_metadata() { return false; }
+};
+
class ShenandoahTraversalClosure : public ShenandoahTraversalSuperClosure {
private:
template <class T>
- inline void do_oop_work(T* p) { work<T, false, false>(p); }
+ inline void do_oop_work(T* p) { work<T, false, false, true>(p); }
public:
ShenandoahTraversalClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
@@ -237,7 +252,7 @@
class ShenandoahTraversalMetadataClosure : public ShenandoahTraversalSuperClosure {
private:
template <class T>
- inline void do_oop_work(T* p) { work<T, false, false>(p); }
+ inline void do_oop_work(T* p) { work<T, false, false, true>(p); }
public:
ShenandoahTraversalMetadataClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
@@ -252,7 +267,7 @@
class ShenandoahTraversalDedupClosure : public ShenandoahTraversalSuperClosure {
private:
template <class T>
- inline void do_oop_work(T* p) { work<T, true, false>(p); }
+ inline void do_oop_work(T* p) { work<T, true, false, true>(p); }
public:
ShenandoahTraversalDedupClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
@@ -267,7 +282,7 @@
class ShenandoahTraversalMetadataDedupClosure : public ShenandoahTraversalSuperClosure {
private:
template <class T>
- inline void do_oop_work(T* p) { work<T, true, false>(p); }
+ inline void do_oop_work(T* p) { work<T, true, false, true>(p); }
public:
ShenandoahTraversalMetadataDedupClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
@@ -282,7 +297,7 @@
class ShenandoahTraversalDegenClosure : public ShenandoahTraversalSuperClosure {
private:
template <class T>
- inline void do_oop_work(T* p) { work<T, false, true>(p); }
+ inline void do_oop_work(T* p) { work<T, false, true, false>(p); }
public:
ShenandoahTraversalDegenClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
@@ -297,7 +312,7 @@
class ShenandoahTraversalMetadataDegenClosure : public ShenandoahTraversalSuperClosure {
private:
template <class T>
- inline void do_oop_work(T* p) { work<T, false, true>(p); }
+ inline void do_oop_work(T* p) { work<T, false, true, false>(p); }
public:
ShenandoahTraversalMetadataDegenClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
@@ -312,7 +327,7 @@
class ShenandoahTraversalDedupDegenClosure : public ShenandoahTraversalSuperClosure {
private:
template <class T>
- inline void do_oop_work(T* p) { work<T, true, true>(p); }
+ inline void do_oop_work(T* p) { work<T, true, true, false>(p); }
public:
ShenandoahTraversalDedupDegenClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
@@ -327,7 +342,7 @@
class ShenandoahTraversalMetadataDedupDegenClosure : public ShenandoahTraversalSuperClosure {
private:
template <class T>
- inline void do_oop_work(T* p) { work<T, true, true>(p); }
+ inline void do_oop_work(T* p) { work<T, true, true, false>(p); }
public:
ShenandoahTraversalMetadataDedupDegenClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
--- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp Mon Oct 21 21:36:26 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp Mon Oct 21 22:44:48 2019 +0200
@@ -38,9 +38,9 @@
_heap->maybe_update_with_forwarded(p);
}
-template <class T, bool STRING_DEDUP, bool DEGEN>
+template <class T, bool STRING_DEDUP, bool DEGEN, bool ATOMIC_UPDATE>
inline void ShenandoahTraversalSuperClosure::work(T* p) {
- _traversal_gc->process_oop<T, STRING_DEDUP, DEGEN>(p, _thread, _queue, _mark_context);
+ _traversal_gc->process_oop<T, STRING_DEDUP, DEGEN, ATOMIC_UPDATE>(p, _thread, _queue, _mark_context);
}
#endif // SHARE_GC_SHENANDOAH_SHENANDOAHOOPCLOSURES_INLINE_HPP
--- a/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp Mon Oct 21 21:36:26 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp Mon Oct 21 22:44:48 2019 +0200
@@ -186,7 +186,7 @@
// Step 1: Process ordinary GC roots.
{
- ShenandoahTraversalClosure roots_cl(q, rp);
+ ShenandoahTraversalRootsClosure roots_cl(q, rp);
ShenandoahMarkCLDClosure cld_cl(&roots_cl);
MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations);
if (unload_classes) {
@@ -266,7 +266,7 @@
// in similar way during nmethod-register process. Therefore, we don't need to rescan code
// roots here.
if (!_heap->is_degenerated_gc_in_progress()) {
- ShenandoahTraversalClosure roots_cl(q, rp);
+ ShenandoahTraversalRootsClosure roots_cl(q, rp);
ShenandoahTraversalSATBThreadsClosure tc(&satb_cl);
if (unload_classes) {
ShenandoahRemarkCLDClosure remark_cld_cl(&roots_cl);
@@ -767,7 +767,7 @@
template <class T>
inline void do_oop_work(T* p) {
- _traversal_gc->process_oop<T, false /* string dedup */, false /* degen */>(p, _thread, _queue, _mark_context);
+ _traversal_gc->process_oop<T, false /* string dedup */, false /* degen */, true /* atomic update */>(p, _thread, _queue, _mark_context);
}
public:
@@ -789,7 +789,7 @@
template <class T>
inline void do_oop_work(T* p) {
- _traversal_gc->process_oop<T, false /* string dedup */, true /* degen */>(p, _thread, _queue, _mark_context);
+ _traversal_gc->process_oop<T, false /* string dedup */, true /* degen */, false /* atomic update */>(p, _thread, _queue, _mark_context);
}
public:
@@ -812,7 +812,7 @@
template <class T>
inline void do_oop_work(T* p) {
ShenandoahEvacOOMScope evac_scope;
- _traversal_gc->process_oop<T, false /* string dedup */, false /* degen */>(p, _thread, _queue, _mark_context);
+ _traversal_gc->process_oop<T, false /* string dedup */, false /* degen */, true /* atomic update */>(p, _thread, _queue, _mark_context);
}
public:
@@ -835,7 +835,7 @@
template <class T>
inline void do_oop_work(T* p) {
ShenandoahEvacOOMScope evac_scope;
- _traversal_gc->process_oop<T, false /* string dedup */, true /* degen */>(p, _thread, _queue, _mark_context);
+ _traversal_gc->process_oop<T, false /* string dedup */, true /* degen */, false /* atomic update */>(p, _thread, _queue, _mark_context);
}
public:
--- a/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.hpp Mon Oct 21 21:36:26 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.hpp Mon Oct 21 22:44:48 2019 +0200
@@ -48,7 +48,7 @@
void concurrent_traversal_collection();
void final_traversal_collection();
- template <class T, bool STRING_DEDUP, bool DEGEN>
+ template <class T, bool STRING_DEDUP, bool DEGEN, bool ATOMIC_UPDATE>
inline void process_oop(T* p, Thread* thread, ShenandoahObjToScanQueue* queue, ShenandoahMarkingContext* const mark_context);
bool check_and_handle_cancelled_gc(ShenandoahTaskTerminator* terminator, bool sts_yield);
--- a/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.inline.hpp Mon Oct 21 21:36:26 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.inline.hpp Mon Oct 21 22:44:48 2019 +0200
@@ -35,12 +35,13 @@
#include "oops/compressedOops.inline.hpp"
#include "oops/oop.inline.hpp"
-template <class T, bool STRING_DEDUP, bool DEGEN>
+template <class T, bool STRING_DEDUP, bool DEGEN, bool ATOMIC_UPDATE>
void ShenandoahTraversalGC::process_oop(T* p, Thread* thread, ShenandoahObjToScanQueue* queue, ShenandoahMarkingContext* const mark_context) {
T o = RawAccess<>::oop_load(p);
if (!CompressedOops::is_null(o)) {
oop obj = CompressedOops::decode_not_null(o);
if (DEGEN) {
+ assert(!ATOMIC_UPDATE, "Degen path assumes non-atomic updates");
oop forw = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
if (obj != forw) {
// Update reference.
@@ -54,7 +55,11 @@
}
shenandoah_assert_forwarded_except(p, obj, _heap->cancelled_gc());
// Update reference.
- ShenandoahHeap::cas_oop(forw, p, obj);
+ if (ATOMIC_UPDATE) {
+ ShenandoahHeap::cas_oop(forw, p, obj);
+ } else {
+ RawAccess<IS_NOT_NULL>::oop_store(p, forw);
+ }
obj = forw;
}