8233061: ZGC: Enforce memory ordering in segmented bit maps
Reviewed-by: pliden, stefank
--- a/src/hotspot/share/gc/z/zLiveMap.cpp Tue Nov 12 10:45:23 2019 -0800
+++ b/src/hotspot/share/gc/z/zLiveMap.cpp Tue Nov 12 20:01:23 2019 +0000
@@ -54,7 +54,9 @@
// Multiple threads can enter here, make sure only one of them
// resets the marking information while the others busy wait.
- for (uint32_t seqnum = _seqnum; seqnum != ZGlobalSeqNum; seqnum = _seqnum) {
+ for (uint32_t seqnum = OrderAccess::load_acquire(&_seqnum);
+ seqnum != ZGlobalSeqNum;
+ seqnum = OrderAccess::load_acquire(&_seqnum)) {
if ((seqnum != seqnum_initializing) &&
(Atomic::cmpxchg(seqnum_initializing, &_seqnum, seqnum) == seqnum)) {
// Reset marking information
@@ -65,13 +67,13 @@
segment_live_bits().clear();
segment_claim_bits().clear();
- // Make sure the newly reset marking information is
- // globally visible before updating the page seqnum.
- OrderAccess::storestore();
+ assert(_seqnum == seqnum_initializing, "Invalid");
- // Update seqnum
- assert(_seqnum == seqnum_initializing, "Invalid");
- _seqnum = ZGlobalSeqNum;
+ // Make sure the newly reset marking information is ordered
+ // before the update of the page seqnum, such that when the
+ // up-to-date seqnum is load acquired, the bit maps will not
+ // contain stale information.
+ OrderAccess::release_store(&_seqnum, ZGlobalSeqNum);
break;
}
@@ -93,10 +95,6 @@
if (!claim_segment(segment)) {
// Already claimed, wait for live bit to be set
while (!is_segment_live(segment)) {
- // Busy wait. The loadload barrier is needed to make
- // sure we re-read the live bit every time we loop.
- OrderAccess::loadload();
-
// Mark reset contention
if (!contention) {
// Count contention once
--- a/src/hotspot/share/gc/z/zLiveMap.inline.hpp Tue Nov 12 10:45:23 2019 -0800
+++ b/src/hotspot/share/gc/z/zLiveMap.inline.hpp Tue Nov 12 20:01:23 2019 +0000
@@ -30,6 +30,7 @@
#include "gc/z/zOop.inline.hpp"
#include "gc/z/zUtils.inline.hpp"
#include "runtime/atomic.hpp"
+#include "runtime/orderAccess.hpp"
#include "utilities/bitMap.inline.hpp"
#include "utilities/debug.hpp"
@@ -38,7 +39,7 @@
}
inline bool ZLiveMap::is_marked() const {
- return _seqnum == ZGlobalSeqNum;
+ return OrderAccess::load_acquire(&_seqnum) == ZGlobalSeqNum;
}
inline uint32_t ZLiveMap::live_objects() const {
@@ -68,15 +69,15 @@
}
inline bool ZLiveMap::is_segment_live(BitMap::idx_t segment) const {
- return segment_live_bits().at(segment);
+ return segment_live_bits().par_at(segment);
}
inline bool ZLiveMap::set_segment_live_atomic(BitMap::idx_t segment) {
- return segment_live_bits().par_set_bit(segment);
+ return segment_live_bits().par_set_bit(segment, memory_order_release);
}
inline bool ZLiveMap::claim_segment(BitMap::idx_t segment) {
- return segment_claim_bits().par_set_bit(segment);
+ return segment_claim_bits().par_set_bit(segment, memory_order_acq_rel);
}
inline BitMap::idx_t ZLiveMap::first_live_segment() const {