111 T o = RawAccess<>::oop_load(p); |
111 T o = RawAccess<>::oop_load(p); |
112 if (!CompressedOops::is_null(o)) { |
112 if (!CompressedOops::is_null(o)) { |
113 oop heap_oop = CompressedOops::decode_not_null(o); |
113 oop heap_oop = CompressedOops::decode_not_null(o); |
114 if (in_collection_set(heap_oop)) { |
114 if (in_collection_set(heap_oop)) { |
115 oop forwarded_oop = ShenandoahBarrierSet::resolve_forwarded_not_null(heap_oop); |
115 oop forwarded_oop = ShenandoahBarrierSet::resolve_forwarded_not_null(heap_oop); |
116 if (oopDesc::equals_raw(forwarded_oop, heap_oop)) { |
116 if (forwarded_oop == heap_oop) { |
117 forwarded_oop = evacuate_object(heap_oop, Thread::current()); |
117 forwarded_oop = evacuate_object(heap_oop, Thread::current()); |
118 } |
118 } |
119 oop prev = atomic_compare_exchange_oop(forwarded_oop, p, heap_oop); |
119 oop prev = cas_oop(forwarded_oop, p, heap_oop); |
120 if (oopDesc::equals_raw(prev, heap_oop)) { |
120 if (prev == heap_oop) { |
121 return forwarded_oop; |
121 return forwarded_oop; |
122 } else { |
122 } else { |
123 return NULL; |
123 return NULL; |
124 } |
124 } |
125 } |
125 } |
127 } else { |
127 } else { |
128 return NULL; |
128 return NULL; |
129 } |
129 } |
130 } |
130 } |
131 |
131 |
132 inline oop ShenandoahHeap::atomic_compare_exchange_oop(oop n, oop* addr, oop c) { |
132 inline oop ShenandoahHeap::cas_oop(oop n, oop* addr, oop c) { |
133 return (oop) Atomic::cmpxchg(n, addr, c); |
133 return (oop) Atomic::cmpxchg(n, addr, c); |
134 } |
134 } |
135 |
135 |
136 inline oop ShenandoahHeap::atomic_compare_exchange_oop(oop n, narrowOop* addr, oop c) { |
136 inline oop ShenandoahHeap::cas_oop(oop n, narrowOop* addr, narrowOop c) { |
|
137 narrowOop val = CompressedOops::encode(n); |
|
138 return CompressedOops::decode((narrowOop) Atomic::cmpxchg(val, addr, c)); |
|
139 } |
|
140 |
|
141 inline oop ShenandoahHeap::cas_oop(oop n, narrowOop* addr, oop c) { |
137 narrowOop cmp = CompressedOops::encode(c); |
142 narrowOop cmp = CompressedOops::encode(c); |
138 narrowOop val = CompressedOops::encode(n); |
143 narrowOop val = CompressedOops::encode(n); |
139 return CompressedOops::decode((narrowOop) Atomic::cmpxchg(val, addr, cmp)); |
144 return CompressedOops::decode((narrowOop) Atomic::cmpxchg(val, addr, cmp)); |
140 } |
145 } |
141 |
146 |
144 shenandoah_assert_not_in_cset_loc_except(p, !is_in(p) || is_full_gc_in_progress() || is_degenerated_gc_in_progress()); |
149 shenandoah_assert_not_in_cset_loc_except(p, !is_in(p) || is_full_gc_in_progress() || is_degenerated_gc_in_progress()); |
145 shenandoah_assert_correct(p, heap_oop); |
150 shenandoah_assert_correct(p, heap_oop); |
146 |
151 |
147 if (in_collection_set(heap_oop)) { |
152 if (in_collection_set(heap_oop)) { |
148 oop forwarded_oop = ShenandoahBarrierSet::resolve_forwarded_not_null(heap_oop); |
153 oop forwarded_oop = ShenandoahBarrierSet::resolve_forwarded_not_null(heap_oop); |
149 if (oopDesc::equals_raw(forwarded_oop, heap_oop)) { |
154 if (forwarded_oop == heap_oop) { |
150 // E.g. during evacuation. |
155 // E.g. during evacuation. |
151 return forwarded_oop; |
156 return forwarded_oop; |
152 } |
157 } |
153 |
158 |
154 shenandoah_assert_forwarded_except(p, heap_oop, is_full_gc_in_progress() || is_degenerated_gc_in_progress()); |
159 shenandoah_assert_forwarded_except(p, heap_oop, is_full_gc_in_progress() || is_degenerated_gc_in_progress()); |
|
160 shenandoah_assert_not_forwarded(p, forwarded_oop); |
155 shenandoah_assert_not_in_cset_except(p, forwarded_oop, cancelled_gc()); |
161 shenandoah_assert_not_in_cset_except(p, forwarded_oop, cancelled_gc()); |
156 |
162 |
157 // If this fails, another thread wrote to p before us, it will be logged in SATB and the |
163 // If this fails, another thread wrote to p before us, it will be logged in SATB and the |
158 // reference be updated later. |
164 // reference be updated later. |
159 oop result = atomic_compare_exchange_oop(forwarded_oop, p, heap_oop); |
165 oop witness = cas_oop(forwarded_oop, p, heap_oop); |
160 |
166 |
161 if (oopDesc::equals_raw(result, heap_oop)) { // CAS successful. |
167 if (witness != heap_oop) { |
|
168 // CAS failed, someone had beat us to it. Normally, we would return the failure witness, |
|
169 // because that would be the proper write of to-space object, enforced by strong barriers. |
|
170 // However, there is a corner case with arraycopy. It can happen that a Java thread |
|
171 // beats us with an arraycopy, which first copies the array, which potentially contains |
|
172 // from-space refs, and only afterwards updates all from-space refs to to-space refs, |
|
173 // which leaves a short window where the new array elements can be from-space. |
|
174 // In this case, we can just resolve the result again. As we resolve, we need to consider |
|
175 // the contended write might have been NULL. |
|
176 oop result = ShenandoahBarrierSet::resolve_forwarded(witness); |
|
177 shenandoah_assert_not_forwarded_except(p, result, (result == NULL)); |
|
178 shenandoah_assert_not_in_cset_except(p, result, (result == NULL) || cancelled_gc()); |
|
179 return result; |
|
180 } else { |
|
181 // Success! We have updated with known to-space copy. We have already asserted it is sane. |
162 return forwarded_oop; |
182 return forwarded_oop; |
163 } else { |
|
164 // Note: we used to assert the following here. This doesn't work because sometimes, during |
|
165 // marking/updating-refs, it can happen that a Java thread beats us with an arraycopy, |
|
166 // which first copies the array, which potentially contains from-space refs, and only afterwards |
|
167 // updates all from-space refs to to-space refs, which leaves a short window where the new array |
|
168 // elements can be from-space. |
|
169 // assert(CompressedOops::is_null(result) || |
|
170 // oopDesc::equals_raw(result, ShenandoahBarrierSet::resolve_oop_static_not_null(result)), |
|
171 // "expect not forwarded"); |
|
172 return NULL; |
|
173 } |
183 } |
174 } else { |
184 } else { |
175 shenandoah_assert_not_forwarded(p, heap_oop); |
185 shenandoah_assert_not_forwarded(p, heap_oop); |
176 return heap_oop; |
186 return heap_oop; |
177 } |
187 } |
272 Copy::aligned_disjoint_words((HeapWord*) p, copy, size); |
282 Copy::aligned_disjoint_words((HeapWord*) p, copy, size); |
273 |
283 |
274 // Try to install the new forwarding pointer. |
284 // Try to install the new forwarding pointer. |
275 oop copy_val = oop(copy); |
285 oop copy_val = oop(copy); |
276 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val); |
286 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val); |
277 if (oopDesc::equals_raw(result, copy_val)) { |
287 if (result == copy_val) { |
278 // Successfully evacuated. Our copy is now the public one! |
288 // Successfully evacuated. Our copy is now the public one! |
279 shenandoah_assert_correct(NULL, copy_val); |
289 shenandoah_assert_correct(NULL, copy_val); |
280 return copy_val; |
290 return copy_val; |
281 } else { |
291 } else { |
282 // Failed to evacuate. We need to deal with the object that is left behind. Since this |
292 // Failed to evacuate. We need to deal with the object that is left behind. Since this |