59 T volatile* dest) const { |
58 T volatile* dest) const { |
60 STATIC_ASSERT(8 == sizeof(T)); |
59 STATIC_ASSERT(8 == sizeof(T)); |
61 (*os::atomic_store_long_func)( |
60 (*os::atomic_store_long_func)( |
62 PrimitiveConversions::cast<int64_t>(store_value), reinterpret_cast<volatile int64_t*>(dest)); |
61 PrimitiveConversions::cast<int64_t>(store_value), reinterpret_cast<volatile int64_t*>(dest)); |
63 } |
62 } |
64 #endif |
|
65 |
63 |
66 // As per atomic.hpp all read-modify-write operations have to provide two-way |
64 // As per atomic.hpp all read-modify-write operations have to provide two-way |
67 // barriers semantics. For AARCH64 we are using load-acquire-with-reservation and |
65 // barriers semantics. |
68 // store-release-with-reservation. While load-acquire combined with store-release |
|
69 // do not generally form two-way barriers, their use with reservations does - the |
|
70 // ARMv8 architecture manual Section F "Barrier Litmus Tests" indicates they |
|
71 // provide sequentially consistent semantics. All we need to add is an explicit |
|
72 // barrier in the failure path of the cmpxchg operations (as these don't execute |
|
73 // the store) - arguably this may be overly cautious as there is a very low |
|
74 // likelihood that the hardware would pull loads/stores into the region guarded |
|
75 // by the reservation. |
|
76 // |
66 // |
77 // For ARMv7 we add explicit barriers in the stubs. |
67 // For ARMv7 we add explicit barriers in the stubs. |
78 |
68 |
79 template<size_t byte_size> |
69 template<size_t byte_size> |
80 struct Atomic::PlatformAdd |
70 struct Atomic::PlatformAdd |
88 template<typename I, typename D> |
78 template<typename I, typename D> |
89 inline D Atomic::PlatformAdd<4>::add_and_fetch(I add_value, D volatile* dest, |
79 inline D Atomic::PlatformAdd<4>::add_and_fetch(I add_value, D volatile* dest, |
90 atomic_memory_order order) const { |
80 atomic_memory_order order) const { |
91 STATIC_ASSERT(4 == sizeof(I)); |
81 STATIC_ASSERT(4 == sizeof(I)); |
92 STATIC_ASSERT(4 == sizeof(D)); |
82 STATIC_ASSERT(4 == sizeof(D)); |
93 #ifdef AARCH64 |
|
94 D val; |
|
95 int tmp; |
|
96 __asm__ volatile( |
|
97 "1:\n\t" |
|
98 " ldaxr %w[val], [%[dest]]\n\t" |
|
99 " add %w[val], %w[val], %w[add_val]\n\t" |
|
100 " stlxr %w[tmp], %w[val], [%[dest]]\n\t" |
|
101 " cbnz %w[tmp], 1b\n\t" |
|
102 : [val] "=&r" (val), [tmp] "=&r" (tmp) |
|
103 : [add_val] "r" (add_value), [dest] "r" (dest) |
|
104 : "memory"); |
|
105 return val; |
|
106 #else |
|
107 return add_using_helper<int32_t>(os::atomic_add_func, add_value, dest); |
83 return add_using_helper<int32_t>(os::atomic_add_func, add_value, dest); |
108 #endif |
|
109 } |
84 } |
110 |
85 |
111 #ifdef AARCH64 |
|
112 template<> |
|
113 template<typename I, typename D> |
|
114 inline D Atomic::PlatformAdd<8>::add_and_fetch(I add_value, D volatile* dest, |
|
115 atomic_memory_order order) const { |
|
116 STATIC_ASSERT(8 == sizeof(I)); |
|
117 STATIC_ASSERT(8 == sizeof(D)); |
|
118 D val; |
|
119 int tmp; |
|
120 __asm__ volatile( |
|
121 "1:\n\t" |
|
122 " ldaxr %[val], [%[dest]]\n\t" |
|
123 " add %[val], %[val], %[add_val]\n\t" |
|
124 " stlxr %w[tmp], %[val], [%[dest]]\n\t" |
|
125 " cbnz %w[tmp], 1b\n\t" |
|
126 : [val] "=&r" (val), [tmp] "=&r" (tmp) |
|
127 : [add_val] "r" (add_value), [dest] "r" (dest) |
|
128 : "memory"); |
|
129 return val; |
|
130 } |
|
131 #endif |
|
132 |
86 |
133 template<> |
87 template<> |
134 template<typename T> |
88 template<typename T> |
135 inline T Atomic::PlatformXchg<4>::operator()(T exchange_value, |
89 inline T Atomic::PlatformXchg<4>::operator()(T exchange_value, |
136 T volatile* dest, |
90 T volatile* dest, |
137 atomic_memory_order order) const { |
91 atomic_memory_order order) const { |
138 STATIC_ASSERT(4 == sizeof(T)); |
92 STATIC_ASSERT(4 == sizeof(T)); |
139 #ifdef AARCH64 |
|
140 T old_val; |
|
141 int tmp; |
|
142 __asm__ volatile( |
|
143 "1:\n\t" |
|
144 " ldaxr %w[old_val], [%[dest]]\n\t" |
|
145 " stlxr %w[tmp], %w[new_val], [%[dest]]\n\t" |
|
146 " cbnz %w[tmp], 1b\n\t" |
|
147 : [old_val] "=&r" (old_val), [tmp] "=&r" (tmp) |
|
148 : [new_val] "r" (exchange_value), [dest] "r" (dest) |
|
149 : "memory"); |
|
150 return old_val; |
|
151 #else |
|
152 return xchg_using_helper<int32_t>(os::atomic_xchg_func, exchange_value, dest); |
93 return xchg_using_helper<int32_t>(os::atomic_xchg_func, exchange_value, dest); |
153 #endif |
|
154 } |
94 } |
155 |
95 |
156 #ifdef AARCH64 |
|
157 template<> |
|
158 template<typename T> |
|
159 inline T Atomic::PlatformXchg<8>::operator()(T exchange_value, |
|
160 T volatile* dest, |
|
161 atomic_memory_order order) const { |
|
162 STATIC_ASSERT(8 == sizeof(T)); |
|
163 T old_val; |
|
164 int tmp; |
|
165 __asm__ volatile( |
|
166 "1:\n\t" |
|
167 " ldaxr %[old_val], [%[dest]]\n\t" |
|
168 " stlxr %w[tmp], %[new_val], [%[dest]]\n\t" |
|
169 " cbnz %w[tmp], 1b\n\t" |
|
170 : [old_val] "=&r" (old_val), [tmp] "=&r" (tmp) |
|
171 : [new_val] "r" (exchange_value), [dest] "r" (dest) |
|
172 : "memory"); |
|
173 return old_val; |
|
174 } |
|
175 #endif // AARCH64 |
|
176 |
96 |
177 // The memory_order parameter is ignored - we always provide the strongest/most-conservative ordering |
97 // The memory_order parameter is ignored - we always provide the strongest/most-conservative ordering |
178 |
98 |
179 // No direct support for cmpxchg of bytes; emulate using int. |
99 // No direct support for cmpxchg of bytes; emulate using int. |
180 template<> |
100 template<> |
181 struct Atomic::PlatformCmpxchg<1> : Atomic::CmpxchgByteUsingInt {}; |
101 struct Atomic::PlatformCmpxchg<1> : Atomic::CmpxchgByteUsingInt {}; |
182 |
102 |
183 #ifndef AARCH64 |
|
184 |
103 |
185 inline int32_t reorder_cmpxchg_func(int32_t exchange_value, |
104 inline int32_t reorder_cmpxchg_func(int32_t exchange_value, |
186 int32_t volatile* dest, |
105 int32_t volatile* dest, |
187 int32_t compare_value) { |
106 int32_t compare_value) { |
188 // Warning: Arguments are swapped to avoid moving them for kernel call |
107 // Warning: Arguments are swapped to avoid moving them for kernel call |
195 assert(VM_Version::supports_cx8(), "Atomic compare and exchange int64_t not supported on this architecture!"); |
114 assert(VM_Version::supports_cx8(), "Atomic compare and exchange int64_t not supported on this architecture!"); |
196 // Warning: Arguments are swapped to avoid moving them for kernel call |
115 // Warning: Arguments are swapped to avoid moving them for kernel call |
197 return (*os::atomic_cmpxchg_long_func)(compare_value, exchange_value, dest); |
116 return (*os::atomic_cmpxchg_long_func)(compare_value, exchange_value, dest); |
198 } |
117 } |
199 |
118 |
200 #endif // !AARCH64 |
|
201 |
119 |
202 template<> |
120 template<> |
203 template<typename T> |
121 template<typename T> |
204 inline T Atomic::PlatformCmpxchg<4>::operator()(T exchange_value, |
122 inline T Atomic::PlatformCmpxchg<4>::operator()(T exchange_value, |
205 T volatile* dest, |
123 T volatile* dest, |
206 T compare_value, |
124 T compare_value, |
207 atomic_memory_order order) const { |
125 atomic_memory_order order) const { |
208 STATIC_ASSERT(4 == sizeof(T)); |
126 STATIC_ASSERT(4 == sizeof(T)); |
209 #ifdef AARCH64 |
|
210 T rv; |
|
211 int tmp; |
|
212 __asm__ volatile( |
|
213 "1:\n\t" |
|
214 " ldaxr %w[rv], [%[dest]]\n\t" |
|
215 " cmp %w[rv], %w[cv]\n\t" |
|
216 " b.ne 2f\n\t" |
|
217 " stlxr %w[tmp], %w[ev], [%[dest]]\n\t" |
|
218 " cbnz %w[tmp], 1b\n\t" |
|
219 " b 3f\n\t" |
|
220 "2:\n\t" |
|
221 " dmb sy\n\t" |
|
222 "3:\n\t" |
|
223 : [rv] "=&r" (rv), [tmp] "=&r" (tmp) |
|
224 : [ev] "r" (exchange_value), [dest] "r" (dest), [cv] "r" (compare_value) |
|
225 : "memory"); |
|
226 return rv; |
|
227 #else |
|
228 return cmpxchg_using_helper<int32_t>(reorder_cmpxchg_func, exchange_value, dest, compare_value); |
127 return cmpxchg_using_helper<int32_t>(reorder_cmpxchg_func, exchange_value, dest, compare_value); |
229 #endif |
|
230 } |
128 } |
231 |
129 |
232 template<> |
130 template<> |
233 template<typename T> |
131 template<typename T> |
234 inline T Atomic::PlatformCmpxchg<8>::operator()(T exchange_value, |
132 inline T Atomic::PlatformCmpxchg<8>::operator()(T exchange_value, |
235 T volatile* dest, |
133 T volatile* dest, |
236 T compare_value, |
134 T compare_value, |
237 atomic_memory_order order) const { |
135 atomic_memory_order order) const { |
238 STATIC_ASSERT(8 == sizeof(T)); |
136 STATIC_ASSERT(8 == sizeof(T)); |
239 #ifdef AARCH64 |
|
240 T rv; |
|
241 int tmp; |
|
242 __asm__ volatile( |
|
243 "1:\n\t" |
|
244 " ldaxr %[rv], [%[dest]]\n\t" |
|
245 " cmp %[rv], %[cv]\n\t" |
|
246 " b.ne 2f\n\t" |
|
247 " stlxr %w[tmp], %[ev], [%[dest]]\n\t" |
|
248 " cbnz %w[tmp], 1b\n\t" |
|
249 " b 3f\n\t" |
|
250 "2:\n\t" |
|
251 " dmb sy\n\t" |
|
252 "3:\n\t" |
|
253 : [rv] "=&r" (rv), [tmp] "=&r" (tmp) |
|
254 : [ev] "r" (exchange_value), [dest] "r" (dest), [cv] "r" (compare_value) |
|
255 : "memory"); |
|
256 return rv; |
|
257 #else |
|
258 return cmpxchg_using_helper<int64_t>(reorder_cmpxchg_long_func, exchange_value, dest, compare_value); |
137 return cmpxchg_using_helper<int64_t>(reorder_cmpxchg_long_func, exchange_value, dest, compare_value); |
259 #endif |
|
260 } |
138 } |
261 |
139 |
262 #endif // OS_CPU_LINUX_ARM_VM_ATOMIC_LINUX_ARM_HPP |
140 #endif // OS_CPU_LINUX_ARM_VM_ATOMIC_LINUX_ARM_HPP |