53 // * atomic_cmpxchg_at: Atomically compare-and-swap a new value at an internal pointer address if previous value matched the compared value. |
53 // * atomic_cmpxchg_at: Atomically compare-and-swap a new value at an internal pointer address if previous value matched the compared value. |
54 // * atomic_xchg: Atomically swap a new value at an address if previous value matched the compared value. |
54 // * atomic_xchg: Atomically swap a new value at an address if previous value matched the compared value. |
55 // * atomic_xchg_at: Atomically swap a new value at an internal pointer address if previous value matched the compared value. |
55 // * atomic_xchg_at: Atomically swap a new value at an internal pointer address if previous value matched the compared value. |
56 // * arraycopy: Copy data from one heap array to another heap array. |
56 // * arraycopy: Copy data from one heap array to another heap array. |
57 // * clone: Clone the contents of an object to a newly allocated object. |
57 // * clone: Clone the contents of an object to a newly allocated object. |
|
58 // * resolve: Resolve a stable to-space invariant oop that is guaranteed not to relocate its payload until a subsequent thread transition. |
58 |
59 |
59 typedef uint64_t DecoratorSet; |
60 typedef uint64_t DecoratorSet; |
60 |
61 |
61 // == Internal Decorators - do not use == |
62 // == Internal Decorators - do not use == |
62 // * INTERNAL_EMPTY: This is the name for the empty decorator set (in absence of other decorators). |
63 // * INTERNAL_EMPTY: This is the name for the empty decorator set (in absence of other decorators). |
67 const DecoratorSet INTERNAL_CONVERT_COMPRESSED_OOP = UCONST64(1) << 1; |
68 const DecoratorSet INTERNAL_CONVERT_COMPRESSED_OOP = UCONST64(1) << 1; |
68 const DecoratorSet INTERNAL_VALUE_IS_OOP = UCONST64(1) << 2; |
69 const DecoratorSet INTERNAL_VALUE_IS_OOP = UCONST64(1) << 2; |
69 |
70 |
70 // == Internal build-time Decorators == |
71 // == Internal build-time Decorators == |
71 // * INTERNAL_BT_BARRIER_ON_PRIMITIVES: This is set in the barrierSetConfig.hpp file. |
72 // * INTERNAL_BT_BARRIER_ON_PRIMITIVES: This is set in the barrierSetConfig.hpp file. |
|
73 // * INTERNAL_BT_TO_SPACE_INVARIANT: This is set in the barrierSetConfig.hpp file iff |
|
74 // no GC is bundled in the build that is to-space invariant. |
72 const DecoratorSet INTERNAL_BT_BARRIER_ON_PRIMITIVES = UCONST64(1) << 3; |
75 const DecoratorSet INTERNAL_BT_BARRIER_ON_PRIMITIVES = UCONST64(1) << 3; |
|
76 const DecoratorSet INTERNAL_BT_TO_SPACE_INVARIANT = UCONST64(1) << 4; |
73 |
77 |
74 // == Internal run-time Decorators == |
78 // == Internal run-time Decorators == |
75 // * INTERNAL_RT_USE_COMPRESSED_OOPS: This decorator will be set in runtime resolved |
79 // * INTERNAL_RT_USE_COMPRESSED_OOPS: This decorator will be set in runtime resolved |
76 // access backends iff UseCompressedOops is true. |
80 // access backends iff UseCompressedOops is true. |
77 const DecoratorSet INTERNAL_RT_USE_COMPRESSED_OOPS = UCONST64(1) << 4; |
81 const DecoratorSet INTERNAL_RT_USE_COMPRESSED_OOPS = UCONST64(1) << 5; |
78 |
82 |
79 const DecoratorSet INTERNAL_DECORATOR_MASK = INTERNAL_CONVERT_COMPRESSED_OOP | INTERNAL_VALUE_IS_OOP | |
83 const DecoratorSet INTERNAL_DECORATOR_MASK = INTERNAL_CONVERT_COMPRESSED_OOP | INTERNAL_VALUE_IS_OOP | |
80 INTERNAL_BT_BARRIER_ON_PRIMITIVES | INTERNAL_RT_USE_COMPRESSED_OOPS; |
84 INTERNAL_BT_BARRIER_ON_PRIMITIVES | INTERNAL_RT_USE_COMPRESSED_OOPS; |
81 |
85 |
82 // == Memory Ordering Decorators == |
86 // == Memory Ordering Decorators == |
136 // === Atomic Xchg === |
140 // === Atomic Xchg === |
137 // * MO_RELAXED: Atomic but relaxed atomic xchg. |
141 // * MO_RELAXED: Atomic but relaxed atomic xchg. |
138 // - Guarantees from MO_RELAXED loads and MO_RELAXED stores hold. |
142 // - Guarantees from MO_RELAXED loads and MO_RELAXED stores hold. |
139 // * MO_SEQ_CST: Sequentially consistent xchg. |
143 // * MO_SEQ_CST: Sequentially consistent xchg. |
140 // - Guarantees from MO_SEQ_CST loads and MO_SEQ_CST stores hold. |
144 // - Guarantees from MO_SEQ_CST loads and MO_SEQ_CST stores hold. |
141 const DecoratorSet MO_UNORDERED = UCONST64(1) << 5; |
145 const DecoratorSet MO_UNORDERED = UCONST64(1) << 6; |
142 const DecoratorSet MO_VOLATILE = UCONST64(1) << 6; |
146 const DecoratorSet MO_VOLATILE = UCONST64(1) << 7; |
143 const DecoratorSet MO_RELAXED = UCONST64(1) << 7; |
147 const DecoratorSet MO_RELAXED = UCONST64(1) << 8; |
144 const DecoratorSet MO_ACQUIRE = UCONST64(1) << 8; |
148 const DecoratorSet MO_ACQUIRE = UCONST64(1) << 9; |
145 const DecoratorSet MO_RELEASE = UCONST64(1) << 9; |
149 const DecoratorSet MO_RELEASE = UCONST64(1) << 10; |
146 const DecoratorSet MO_SEQ_CST = UCONST64(1) << 10; |
150 const DecoratorSet MO_SEQ_CST = UCONST64(1) << 11; |
147 const DecoratorSet MO_DECORATOR_MASK = MO_UNORDERED | MO_VOLATILE | MO_RELAXED | |
151 const DecoratorSet MO_DECORATOR_MASK = MO_UNORDERED | MO_VOLATILE | MO_RELAXED | |
148 MO_ACQUIRE | MO_RELEASE | MO_SEQ_CST; |
152 MO_ACQUIRE | MO_RELEASE | MO_SEQ_CST; |
149 |
153 |
150 // === Barrier Strength Decorators === |
154 // === Barrier Strength Decorators === |
151 // * AS_RAW: The access will translate into a raw memory access, hence ignoring all semantic concerns |
155 // * AS_RAW: The access will translate into a raw memory access, hence ignoring all semantic concerns |
164 // extreme caution in isolated scopes. |
168 // extreme caution in isolated scopes. |
165 // * AS_NORMAL: The accesses will be resolved to an accessor on the BarrierSet class, giving the |
169 // * AS_NORMAL: The accesses will be resolved to an accessor on the BarrierSet class, giving the |
166 // responsibility of performing the access and what barriers to be performed to the GC. This is the default. |
170 // responsibility of performing the access and what barriers to be performed to the GC. This is the default. |
167 // Note that primitive accesses will only be resolved on the barrier set if the appropriate build-time |
171 // Note that primitive accesses will only be resolved on the barrier set if the appropriate build-time |
168 // decorator for enabling primitive barriers is enabled for the build. |
172 // decorator for enabling primitive barriers is enabled for the build. |
169 const DecoratorSet AS_RAW = UCONST64(1) << 11; |
173 const DecoratorSet AS_RAW = UCONST64(1) << 12; |
170 const DecoratorSet AS_DEST_NOT_INITIALIZED = UCONST64(1) << 12; |
174 const DecoratorSet AS_DEST_NOT_INITIALIZED = UCONST64(1) << 13; |
171 const DecoratorSet AS_NO_KEEPALIVE = UCONST64(1) << 13; |
175 const DecoratorSet AS_NO_KEEPALIVE = UCONST64(1) << 14; |
172 const DecoratorSet AS_NORMAL = UCONST64(1) << 14; |
176 const DecoratorSet AS_NORMAL = UCONST64(1) << 15; |
173 const DecoratorSet AS_DECORATOR_MASK = AS_RAW | AS_DEST_NOT_INITIALIZED | |
177 const DecoratorSet AS_DECORATOR_MASK = AS_RAW | AS_DEST_NOT_INITIALIZED | |
174 AS_NO_KEEPALIVE | AS_NORMAL; |
178 AS_NO_KEEPALIVE | AS_NORMAL; |
175 |
179 |
176 // === Reference Strength Decorators === |
180 // === Reference Strength Decorators === |
177 // These decorators only apply to accesses on oop-like types (oop/narrowOop). |
181 // These decorators only apply to accesses on oop-like types (oop/narrowOop). |
180 // * ON_PHANTOM_OOP_REF: The memory access is performed on a phantomly reachable reference. |
184 // * ON_PHANTOM_OOP_REF: The memory access is performed on a phantomly reachable reference. |
181 // This is the same ring of strength as jweak and weak oops in the VM. |
185 // This is the same ring of strength as jweak and weak oops in the VM. |
182 // * ON_UNKNOWN_OOP_REF: The memory access is performed on a reference of unknown strength. |
186 // * ON_UNKNOWN_OOP_REF: The memory access is performed on a reference of unknown strength. |
183 // This could for example come from the unsafe API. |
187 // This could for example come from the unsafe API. |
184 // * Default (no explicit reference strength specified): ON_STRONG_OOP_REF |
188 // * Default (no explicit reference strength specified): ON_STRONG_OOP_REF |
185 const DecoratorSet ON_STRONG_OOP_REF = UCONST64(1) << 15; |
189 const DecoratorSet ON_STRONG_OOP_REF = UCONST64(1) << 16; |
186 const DecoratorSet ON_WEAK_OOP_REF = UCONST64(1) << 16; |
190 const DecoratorSet ON_WEAK_OOP_REF = UCONST64(1) << 17; |
187 const DecoratorSet ON_PHANTOM_OOP_REF = UCONST64(1) << 17; |
191 const DecoratorSet ON_PHANTOM_OOP_REF = UCONST64(1) << 18; |
188 const DecoratorSet ON_UNKNOWN_OOP_REF = UCONST64(1) << 18; |
192 const DecoratorSet ON_UNKNOWN_OOP_REF = UCONST64(1) << 19; |
189 const DecoratorSet ON_DECORATOR_MASK = ON_STRONG_OOP_REF | ON_WEAK_OOP_REF | |
193 const DecoratorSet ON_DECORATOR_MASK = ON_STRONG_OOP_REF | ON_WEAK_OOP_REF | |
190 ON_PHANTOM_OOP_REF | ON_UNKNOWN_OOP_REF; |
194 ON_PHANTOM_OOP_REF | ON_UNKNOWN_OOP_REF; |
191 |
195 |
192 // === Access Location === |
196 // === Access Location === |
193 // Accesses can take place in, e.g. the heap, old or young generation and different native roots. |
197 // Accesses can take place in, e.g. the heap, old or young generation and different native roots. |
198 // for some GCs, and implies that it is an IN_HEAP. |
202 // for some GCs, and implies that it is an IN_HEAP. |
199 // * IN_ROOT: The access is performed in an off-heap data structure pointing into the Java heap. |
203 // * IN_ROOT: The access is performed in an off-heap data structure pointing into the Java heap. |
200 // * IN_CONCURRENT_ROOT: The access is performed in an off-heap data structure pointing into the Java heap, |
204 // * IN_CONCURRENT_ROOT: The access is performed in an off-heap data structure pointing into the Java heap, |
201 // but is notably not scanned during safepoints. This is sometimes a special case for some GCs and |
205 // but is notably not scanned during safepoints. This is sometimes a special case for some GCs and |
202 // implies that it is also an IN_ROOT. |
206 // implies that it is also an IN_ROOT. |
203 const DecoratorSet IN_HEAP = UCONST64(1) << 19; |
207 const DecoratorSet IN_HEAP = UCONST64(1) << 20; |
204 const DecoratorSet IN_HEAP_ARRAY = UCONST64(1) << 20; |
208 const DecoratorSet IN_HEAP_ARRAY = UCONST64(1) << 21; |
205 const DecoratorSet IN_ROOT = UCONST64(1) << 21; |
209 const DecoratorSet IN_ROOT = UCONST64(1) << 22; |
206 const DecoratorSet IN_CONCURRENT_ROOT = UCONST64(1) << 22; |
210 const DecoratorSet IN_CONCURRENT_ROOT = UCONST64(1) << 23; |
207 const DecoratorSet IN_ARCHIVE_ROOT = UCONST64(1) << 23; |
211 const DecoratorSet IN_ARCHIVE_ROOT = UCONST64(1) << 24; |
208 const DecoratorSet IN_DECORATOR_MASK = IN_HEAP | IN_HEAP_ARRAY | |
212 const DecoratorSet IN_DECORATOR_MASK = IN_HEAP | IN_HEAP_ARRAY | |
209 IN_ROOT | IN_CONCURRENT_ROOT | |
213 IN_ROOT | IN_CONCURRENT_ROOT | |
210 IN_ARCHIVE_ROOT; |
214 IN_ARCHIVE_ROOT; |
211 |
215 |
212 // == Value Decorators == |
216 // == Value Decorators == |
213 // * OOP_NOT_NULL: This property can make certain barriers faster such as compressing oops. |
217 // * OOP_NOT_NULL: This property can make certain barriers faster such as compressing oops. |
214 const DecoratorSet OOP_NOT_NULL = UCONST64(1) << 24; |
218 const DecoratorSet OOP_NOT_NULL = UCONST64(1) << 25; |
215 const DecoratorSet OOP_DECORATOR_MASK = OOP_NOT_NULL; |
219 const DecoratorSet OOP_DECORATOR_MASK = OOP_NOT_NULL; |
216 |
220 |
217 // == Arraycopy Decorators == |
221 // == Arraycopy Decorators == |
218 // * ARRAYCOPY_CHECKCAST: This property means that the class of the objects in source |
222 // * ARRAYCOPY_CHECKCAST: This property means that the class of the objects in source |
219 // are not guaranteed to be subclasses of the class of the destination array. This requires |
223 // are not guaranteed to be subclasses of the class of the destination array. This requires |
222 // * ARRAYCOPY_DISJOINT: This property means that it is known that the two array ranges |
226 // * ARRAYCOPY_DISJOINT: This property means that it is known that the two array ranges |
223 // are disjoint. |
227 // are disjoint. |
224 // * ARRAYCOPY_ARRAYOF: The copy is in the arrayof form. |
228 // * ARRAYCOPY_ARRAYOF: The copy is in the arrayof form. |
225 // * ARRAYCOPY_ATOMIC: The accesses have to be atomic over the size of its elements. |
229 // * ARRAYCOPY_ATOMIC: The accesses have to be atomic over the size of its elements. |
226 // * ARRAYCOPY_ALIGNED: The accesses have to be aligned on a HeapWord. |
230 // * ARRAYCOPY_ALIGNED: The accesses have to be aligned on a HeapWord. |
227 const DecoratorSet ARRAYCOPY_CHECKCAST = UCONST64(1) << 25; |
231 const DecoratorSet ARRAYCOPY_CHECKCAST = UCONST64(1) << 26; |
228 const DecoratorSet ARRAYCOPY_DISJOINT = UCONST64(1) << 26; |
232 const DecoratorSet ARRAYCOPY_DISJOINT = UCONST64(1) << 27; |
229 const DecoratorSet ARRAYCOPY_ARRAYOF = UCONST64(1) << 27; |
233 const DecoratorSet ARRAYCOPY_ARRAYOF = UCONST64(1) << 28; |
230 const DecoratorSet ARRAYCOPY_ATOMIC = UCONST64(1) << 28; |
234 const DecoratorSet ARRAYCOPY_ATOMIC = UCONST64(1) << 29; |
231 const DecoratorSet ARRAYCOPY_ALIGNED = UCONST64(1) << 29; |
235 const DecoratorSet ARRAYCOPY_ALIGNED = UCONST64(1) << 30; |
232 const DecoratorSet ARRAYCOPY_DECORATOR_MASK = ARRAYCOPY_CHECKCAST | ARRAYCOPY_DISJOINT | |
236 const DecoratorSet ARRAYCOPY_DECORATOR_MASK = ARRAYCOPY_CHECKCAST | ARRAYCOPY_DISJOINT | |
233 ARRAYCOPY_DISJOINT | ARRAYCOPY_ARRAYOF | |
237 ARRAYCOPY_DISJOINT | ARRAYCOPY_ARRAYOF | |
234 ARRAYCOPY_ATOMIC | ARRAYCOPY_ALIGNED; |
238 ARRAYCOPY_ATOMIC | ARRAYCOPY_ALIGNED; |
235 |
239 |
236 // The HasDecorator trait can help at compile-time determining whether a decorator set |
240 // The HasDecorator trait can help at compile-time determining whether a decorator set |
295 bool arraycopy(arrayOop src_obj, arrayOop dst_obj, T *src, T *dst, size_t length); |
299 bool arraycopy(arrayOop src_obj, arrayOop dst_obj, T *src, T *dst, size_t length); |
296 |
300 |
297 template <DecoratorSet decorators> |
301 template <DecoratorSet decorators> |
298 void clone(oop src, oop dst, size_t size); |
302 void clone(oop src, oop dst, size_t size); |
299 |
303 |
|
304 template <DecoratorSet decorators> |
|
305 oop resolve(oop src); |
|
306 |
300 // Infer the type that should be returned from a load. |
307 // Infer the type that should be returned from a load. |
301 template <typename P, DecoratorSet decorators> |
308 template <typename P, DecoratorSet decorators> |
302 class LoadProxy: public StackObj { |
309 class LoadProxy: public StackObj { |
303 private: |
310 private: |
304 P *const _addr; |
311 P *const _addr; |
498 verify_oop_decorators<atomic_xchg_mo_decorators>(); |
505 verify_oop_decorators<atomic_xchg_mo_decorators>(); |
499 typedef typename AccessInternal::OopOrNarrowOop<T>::type OopType; |
506 typedef typename AccessInternal::OopOrNarrowOop<T>::type OopType; |
500 OopType new_oop_value = new_value; |
507 OopType new_oop_value = new_value; |
501 return AccessInternal::atomic_xchg<decorators | INTERNAL_VALUE_IS_OOP>(new_oop_value, addr); |
508 return AccessInternal::atomic_xchg<decorators | INTERNAL_VALUE_IS_OOP>(new_oop_value, addr); |
502 } |
509 } |
|
510 |
|
511 static oop resolve(oop obj) { |
|
512 verify_decorators<INTERNAL_EMPTY>(); |
|
513 return AccessInternal::resolve<decorators>(obj); |
|
514 } |
503 }; |
515 }; |
504 |
516 |
505 // Helper for performing raw accesses (knows only of memory ordering |
517 // Helper for performing raw accesses (knows only of memory ordering |
506 // atomicity decorators as well as compressed oops) |
518 // atomicity decorators as well as compressed oops) |
507 template <DecoratorSet decorators = INTERNAL_EMPTY> |
519 template <DecoratorSet decorators = INTERNAL_EMPTY> |