--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp Thu Oct 17 20:27:44 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp Thu Oct 17 20:53:35 2019 +0100
@@ -113,11 +113,11 @@
oop heap_oop = CompressedOops::decode_not_null(o);
if (in_collection_set(heap_oop)) {
oop forwarded_oop = ShenandoahBarrierSet::resolve_forwarded_not_null(heap_oop);
- if (oopDesc::equals_raw(forwarded_oop, heap_oop)) {
+ if (forwarded_oop == heap_oop) {
forwarded_oop = evacuate_object(heap_oop, Thread::current());
}
- oop prev = atomic_compare_exchange_oop(forwarded_oop, p, heap_oop);
- if (oopDesc::equals_raw(prev, heap_oop)) {
+ oop prev = cas_oop(forwarded_oop, p, heap_oop);
+ if (prev == heap_oop) {
return forwarded_oop;
} else {
return NULL;
@@ -129,11 +129,16 @@
}
}
-inline oop ShenandoahHeap::atomic_compare_exchange_oop(oop n, oop* addr, oop c) {
+inline oop ShenandoahHeap::cas_oop(oop n, oop* addr, oop c) {
return (oop) Atomic::cmpxchg(n, addr, c);
}
-inline oop ShenandoahHeap::atomic_compare_exchange_oop(oop n, narrowOop* addr, oop c) {
+inline oop ShenandoahHeap::cas_oop(oop n, narrowOop* addr, narrowOop c) {
+ narrowOop val = CompressedOops::encode(n);
+ return CompressedOops::decode((narrowOop) Atomic::cmpxchg(val, addr, c));
+}
+
+inline oop ShenandoahHeap::cas_oop(oop n, narrowOop* addr, oop c) {
narrowOop cmp = CompressedOops::encode(c);
narrowOop val = CompressedOops::encode(n);
return CompressedOops::decode((narrowOop) Atomic::cmpxchg(val, addr, cmp));
@@ -146,30 +151,35 @@
if (in_collection_set(heap_oop)) {
oop forwarded_oop = ShenandoahBarrierSet::resolve_forwarded_not_null(heap_oop);
- if (oopDesc::equals_raw(forwarded_oop, heap_oop)) {
+ if (forwarded_oop == heap_oop) {
// E.g. during evacuation.
return forwarded_oop;
}
shenandoah_assert_forwarded_except(p, heap_oop, is_full_gc_in_progress() || is_degenerated_gc_in_progress());
+ shenandoah_assert_not_forwarded(p, forwarded_oop);
shenandoah_assert_not_in_cset_except(p, forwarded_oop, cancelled_gc());
// If this fails, another thread wrote to p before us, it will be logged in SATB and the
// reference be updated later.
- oop result = atomic_compare_exchange_oop(forwarded_oop, p, heap_oop);
+ oop witness = cas_oop(forwarded_oop, p, heap_oop);
- if (oopDesc::equals_raw(result, heap_oop)) { // CAS successful.
- return forwarded_oop;
+ if (witness != heap_oop) {
+ // CAS failed, someone had beat us to it. Normally, we would return the failure witness,
+ // because that would be the proper write of to-space object, enforced by strong barriers.
+ // However, there is a corner case with arraycopy. It can happen that a Java thread
+ // beats us with an arraycopy, which first copies the array, which potentially contains
+ // from-space refs, and only afterwards updates all from-space refs to to-space refs,
+ // which leaves a short window where the new array elements can be from-space.
+ // In this case, we can just resolve the result again. As we resolve, we need to consider
+ // the contended write might have been NULL.
+ oop result = ShenandoahBarrierSet::resolve_forwarded(witness);
+ shenandoah_assert_not_forwarded_except(p, result, (result == NULL));
+ shenandoah_assert_not_in_cset_except(p, result, (result == NULL) || cancelled_gc());
+ return result;
} else {
- // Note: we used to assert the following here. This doesn't work because sometimes, during
- // marking/updating-refs, it can happen that a Java thread beats us with an arraycopy,
- // which first copies the array, which potentially contains from-space refs, and only afterwards
- // updates all from-space refs to to-space refs, which leaves a short window where the new array
- // elements can be from-space.
- // assert(CompressedOops::is_null(result) ||
- // oopDesc::equals_raw(result, ShenandoahBarrierSet::resolve_oop_static_not_null(result)),
- // "expect not forwarded");
- return NULL;
+ // Success! We have updated with known to-space copy. We have already asserted it is sane.
+ return forwarded_oop;
}
} else {
shenandoah_assert_not_forwarded(p, heap_oop);
@@ -274,7 +284,7 @@
// Try to install the new forwarding pointer.
oop copy_val = oop(copy);
oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
- if (oopDesc::equals_raw(result, copy_val)) {
+ if (result == copy_val) {
// Successfully evacuated. Our copy is now the public one!
shenandoah_assert_correct(NULL, copy_val);
return copy_val;