46 } else { |
46 } else { |
47 return p; |
47 return p; |
48 } |
48 } |
49 } |
49 } |
50 |
50 |
|
51 inline void ShenandoahBarrierSet::enqueue(oop obj) { |
|
52 shenandoah_assert_not_forwarded_if(NULL, obj, _heap->is_concurrent_traversal_in_progress()); |
|
53 assert(_satb_mark_queue_set.is_active(), "only get here when SATB active"); |
|
54 |
|
55 // Filter marked objects before hitting the SATB queues. The same predicate would |
|
56 // be used by SATBMQ::filter to eliminate already marked objects downstream, but |
|
57 // filtering here helps to avoid wasteful SATB queueing work to begin with. |
|
58 if (!_heap->requires_marking<false>(obj)) return; |
|
59 |
|
60 ShenandoahThreadLocalData::satb_mark_queue(Thread::current()).enqueue_known_active(obj); |
|
61 } |
|
62 |
|
63 template <DecoratorSet decorators, typename T> |
|
64 inline void ShenandoahBarrierSet::satb_barrier(T *field) { |
|
65 if (HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value || |
|
66 HasDecorator<decorators, AS_NO_KEEPALIVE>::value) { |
|
67 return; |
|
68 } |
|
69 if (ShenandoahSATBBarrier && _heap->is_concurrent_mark_in_progress()) { |
|
70 T heap_oop = RawAccess<>::oop_load(field); |
|
71 if (!CompressedOops::is_null(heap_oop)) { |
|
72 enqueue(CompressedOops::decode(heap_oop)); |
|
73 } |
|
74 } |
|
75 } |
|
76 |
|
77 inline void ShenandoahBarrierSet::satb_enqueue(oop value) { |
|
78 assert(value != NULL, "checked before"); |
|
79 if (ShenandoahSATBBarrier && _heap->is_concurrent_mark_in_progress()) { |
|
80 enqueue(value); |
|
81 } |
|
82 } |
|
83 |
|
84 inline void ShenandoahBarrierSet::storeval_barrier(oop obj) { |
|
85 if (obj != NULL && ShenandoahStoreValEnqueueBarrier && _heap->is_concurrent_traversal_in_progress()) { |
|
86 enqueue(obj); |
|
87 } |
|
88 } |
|
89 |
|
90 inline void ShenandoahBarrierSet::keep_alive_barrier(oop value) { |
|
91 assert(value != NULL, "checked before"); |
|
92 if (ShenandoahKeepAliveBarrier && _heap->is_concurrent_mark_in_progress()) { |
|
93 enqueue(value); |
|
94 } |
|
95 } |
|
96 |
|
97 inline void ShenandoahBarrierSet::keep_alive_if_weak(DecoratorSet decorators, oop value) { |
|
98 assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Reference strength must be known"); |
|
99 const bool on_strong_oop_ref = (decorators & ON_STRONG_OOP_REF) != 0; |
|
100 const bool peek = (decorators & AS_NO_KEEPALIVE) != 0; |
|
101 if (!peek && !on_strong_oop_ref) { |
|
102 keep_alive_barrier(value); |
|
103 } |
|
104 } |
|
105 |
|
106 template <DecoratorSet decorators> |
|
107 inline void ShenandoahBarrierSet::keep_alive_if_weak(oop value) { |
|
108 assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Reference strength must be known"); |
|
109 if (!HasDecorator<decorators, ON_STRONG_OOP_REF>::value && |
|
110 !HasDecorator<decorators, AS_NO_KEEPALIVE>::value) { |
|
111 keep_alive_barrier(value); |
|
112 } |
|
113 } |
|
114 |
|
115 template <DecoratorSet decorators, typename BarrierSetT> |
|
116 template <typename T> |
|
117 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(T* addr) { |
|
118 oop value = Raw::oop_load_not_in_heap(addr); |
|
119 if (value != NULL) { |
|
120 ShenandoahBarrierSet *const bs = ShenandoahBarrierSet::barrier_set(); |
|
121 value = bs->load_reference_barrier_native(value, addr); |
|
122 bs->keep_alive_if_weak<decorators>(value); |
|
123 } |
|
124 return value; |
|
125 } |
|
126 |
51 template <DecoratorSet decorators, typename BarrierSetT> |
127 template <DecoratorSet decorators, typename BarrierSetT> |
52 template <typename T> |
128 template <typename T> |
53 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap(T* addr) { |
129 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap(T* addr) { |
54 oop value = Raw::oop_load_in_heap(addr); |
130 oop value = Raw::oop_load_in_heap(addr); |
55 value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(value); |
131 if (value != NULL) { |
56 keep_alive_if_weak(decorators, value); |
132 ShenandoahBarrierSet *const bs = ShenandoahBarrierSet::barrier_set(); |
|
133 value = bs->load_reference_barrier_not_null(value); |
|
134 bs->keep_alive_if_weak<decorators>(value); |
|
135 } |
57 return value; |
136 return value; |
58 } |
137 } |
59 |
138 |
60 template <DecoratorSet decorators, typename BarrierSetT> |
139 template <DecoratorSet decorators, typename BarrierSetT> |
61 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap_at(oop base, ptrdiff_t offset) { |
140 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap_at(oop base, ptrdiff_t offset) { |
62 oop value = Raw::oop_load_in_heap_at(base, offset); |
141 oop value = Raw::oop_load_in_heap_at(base, offset); |
63 value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(value); |
142 if (value != NULL) { |
64 keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), value); |
143 ShenandoahBarrierSet *const bs = ShenandoahBarrierSet::barrier_set(); |
|
144 value = bs->load_reference_barrier_not_null(value); |
|
145 bs->keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), |
|
146 value); |
|
147 } |
65 return value; |
148 return value; |
66 } |
149 } |
67 |
150 |
68 template <DecoratorSet decorators, typename BarrierSetT> |
151 template <DecoratorSet decorators, typename BarrierSetT> |
69 template <typename T> |
152 template <typename T> |
70 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(T* addr) { |
153 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_not_in_heap(T* addr, oop value) { |
71 oop value = Raw::oop_load_not_in_heap(addr); |
154 shenandoah_assert_marked_if(NULL, value, !CompressedOops::is_null(value) && ShenandoahHeap::heap()->is_evacuation_in_progress()); |
72 value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier_native(value, addr); |
155 ShenandoahBarrierSet* const bs = ShenandoahBarrierSet::barrier_set(); |
73 keep_alive_if_weak(decorators, value); |
156 bs->storeval_barrier(value); |
74 return value; |
157 bs->satb_barrier<decorators>(addr); |
|
158 Raw::oop_store(addr, value); |
75 } |
159 } |
76 |
160 |
77 template <DecoratorSet decorators, typename BarrierSetT> |
161 template <DecoratorSet decorators, typename BarrierSetT> |
78 template <typename T> |
162 template <typename T> |
79 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap(T* addr, oop value) { |
163 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap(T* addr, oop value) { |
80 ShenandoahBarrierSet::barrier_set()->storeval_barrier(value); |
164 shenandoah_assert_not_in_cset_loc_except(addr, ShenandoahHeap::heap()->cancelled_gc()); |
81 const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0; |
165 shenandoah_assert_not_forwarded_except (addr, value, value == NULL || ShenandoahHeap::heap()->cancelled_gc() || !ShenandoahHeap::heap()->is_concurrent_mark_in_progress()); |
82 if (keep_alive) { |
166 shenandoah_assert_not_in_cset_except (addr, value, value == NULL || ShenandoahHeap::heap()->cancelled_gc() || !ShenandoahHeap::heap()->is_concurrent_mark_in_progress()); |
83 ShenandoahBarrierSet::barrier_set()->write_ref_field_pre_work(addr, value); |
167 |
84 } |
168 oop_store_not_in_heap(addr, value); |
85 Raw::oop_store_in_heap(addr, value); |
|
86 } |
169 } |
87 |
170 |
88 template <DecoratorSet decorators, typename BarrierSetT> |
171 template <DecoratorSet decorators, typename BarrierSetT> |
89 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) { |
172 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) { |
90 oop_store_in_heap(AccessInternal::oop_field_addr<decorators>(base, offset), value); |
173 oop_store_in_heap(AccessInternal::oop_field_addr<decorators>(base, offset), value); |
91 } |
174 } |
92 |
175 |
93 template <DecoratorSet decorators, typename BarrierSetT> |
176 template <DecoratorSet decorators, typename BarrierSetT> |
94 template <typename T> |
177 template <typename T> |
95 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_not_in_heap(T* addr, oop value) { |
|
96 shenandoah_assert_marked_if(NULL, value, !CompressedOops::is_null(value) && ShenandoahHeap::heap()->is_evacuation_in_progress()); |
|
97 Raw::oop_store(addr, value); |
|
98 } |
|
99 |
|
100 template <DecoratorSet decorators, typename BarrierSetT> |
|
101 template <typename T> |
|
102 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_not_in_heap(T* addr, oop compare_value, oop new_value) { |
178 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_not_in_heap(T* addr, oop compare_value, oop new_value) { |
|
179 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set(); |
|
180 bs->storeval_barrier(new_value); |
|
181 |
103 oop res; |
182 oop res; |
104 oop expected = compare_value; |
183 oop expected = compare_value; |
105 do { |
184 do { |
106 compare_value = expected; |
185 compare_value = expected; |
107 res = Raw::oop_atomic_cmpxchg(addr, compare_value, new_value); |
186 res = Raw::oop_atomic_cmpxchg(addr, compare_value, new_value); |
108 expected = res; |
187 expected = res; |
109 } while ((compare_value != expected) && (resolve_forwarded(compare_value) == resolve_forwarded(expected))); |
188 } while ((compare_value != expected) && (resolve_forwarded(compare_value) == resolve_forwarded(expected))); |
|
189 |
|
190 // Note: We don't need a keep-alive-barrier here. We already enqueue any loaded reference for SATB anyway, |
|
191 // because it must be the previous value. |
110 if (res != NULL) { |
192 if (res != NULL) { |
111 return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(res); |
193 res = ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(res); |
112 } else { |
194 bs->satb_enqueue(res); |
113 return res; |
195 } |
114 } |
196 return res; |
115 } |
|
116 |
|
117 template <DecoratorSet decorators, typename BarrierSetT> |
|
118 template <typename T> |
|
119 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap_impl(T* addr, oop compare_value, oop new_value) { |
|
120 ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value); |
|
121 oop result = oop_atomic_cmpxchg_not_in_heap(addr, compare_value, new_value); |
|
122 const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0; |
|
123 if (keep_alive && ShenandoahSATBBarrier && !CompressedOops::is_null(result) && |
|
124 (result == compare_value) && |
|
125 ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) { |
|
126 ShenandoahBarrierSet::barrier_set()->enqueue(result); |
|
127 } |
|
128 return result; |
|
129 } |
197 } |
130 |
198 |
131 template <DecoratorSet decorators, typename BarrierSetT> |
199 template <DecoratorSet decorators, typename BarrierSetT> |
132 template <typename T> |
200 template <typename T> |
133 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(T* addr, oop compare_value, oop new_value) { |
201 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(T* addr, oop compare_value, oop new_value) { |
134 oop result = oop_atomic_cmpxchg_in_heap_impl(addr, compare_value, new_value); |
202 return oop_atomic_cmpxchg_not_in_heap(addr, compare_value, new_value); |
135 keep_alive_if_weak(decorators, result); |
|
136 return result; |
|
137 } |
203 } |
138 |
204 |
139 template <DecoratorSet decorators, typename BarrierSetT> |
205 template <DecoratorSet decorators, typename BarrierSetT> |
140 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap_at(oop base, ptrdiff_t offset, oop compare_value, oop new_value) { |
206 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap_at(oop base, ptrdiff_t offset, oop compare_value, oop new_value) { |
141 oop result = oop_atomic_cmpxchg_in_heap_impl(AccessInternal::oop_field_addr<decorators>(base, offset), compare_value, new_value); |
207 return oop_atomic_cmpxchg_in_heap(AccessInternal::oop_field_addr<decorators>(base, offset), compare_value, new_value); |
142 keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), result); |
|
143 return result; |
|
144 } |
208 } |
145 |
209 |
146 template <DecoratorSet decorators, typename BarrierSetT> |
210 template <DecoratorSet decorators, typename BarrierSetT> |
147 template <typename T> |
211 template <typename T> |
148 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_not_in_heap(T* addr, oop new_value) { |
212 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_not_in_heap(T* addr, oop new_value) { |
|
213 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set(); |
|
214 bs->storeval_barrier(new_value); |
|
215 |
149 oop previous = Raw::oop_atomic_xchg(addr, new_value); |
216 oop previous = Raw::oop_atomic_xchg(addr, new_value); |
|
217 |
|
218 // Note: We don't need a keep-alive-barrier here. We already enqueue any loaded reference for SATB anyway, |
|
219 // because it must be the previous value. |
150 if (previous != NULL) { |
220 if (previous != NULL) { |
151 return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(previous); |
221 previous = ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(previous); |
152 } else { |
222 bs->satb_enqueue(previous); |
153 return previous; |
223 } |
154 } |
224 return previous; |
155 } |
|
156 |
|
157 template <DecoratorSet decorators, typename BarrierSetT> |
|
158 template <typename T> |
|
159 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_impl(T* addr, oop new_value) { |
|
160 ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value); |
|
161 oop result = oop_atomic_xchg_not_in_heap(addr, new_value); |
|
162 const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0; |
|
163 if (keep_alive && ShenandoahSATBBarrier && !CompressedOops::is_null(result) && |
|
164 ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) { |
|
165 ShenandoahBarrierSet::barrier_set()->enqueue(result); |
|
166 } |
|
167 return result; |
|
168 } |
225 } |
169 |
226 |
170 template <DecoratorSet decorators, typename BarrierSetT> |
227 template <DecoratorSet decorators, typename BarrierSetT> |
171 template <typename T> |
228 template <typename T> |
172 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(T* addr, oop new_value) { |
229 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(T* addr, oop new_value) { |
173 oop result = oop_atomic_xchg_in_heap_impl(addr, new_value); |
230 return oop_atomic_xchg_not_in_heap(addr, new_value); |
174 keep_alive_if_weak(addr, result); |
|
175 return result; |
|
176 } |
231 } |
177 |
232 |
178 template <DecoratorSet decorators, typename BarrierSetT> |
233 template <DecoratorSet decorators, typename BarrierSetT> |
179 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_at(oop base, ptrdiff_t offset, oop new_value) { |
234 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_at(oop base, ptrdiff_t offset, oop new_value) { |
180 oop result = oop_atomic_xchg_in_heap_impl(AccessInternal::oop_field_addr<decorators>(base, offset), new_value); |
235 return oop_atomic_xchg_in_heap(AccessInternal::oop_field_addr<decorators>(base, offset), new_value); |
181 keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), result); |
|
182 return result; |
|
183 } |
236 } |
184 |
237 |
185 // Clone barrier support |
238 // Clone barrier support |
186 template <DecoratorSet decorators, typename BarrierSetT> |
239 template <DecoratorSet decorators, typename BarrierSetT> |
187 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) { |
240 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) { |