101 template <DecoratorSet decorators, typename T> |
101 template <DecoratorSet decorators, typename T> |
102 struct AccessFunctionTypes { |
102 struct AccessFunctionTypes { |
103 typedef T (*load_at_func_t)(oop base, ptrdiff_t offset); |
103 typedef T (*load_at_func_t)(oop base, ptrdiff_t offset); |
104 typedef void (*store_at_func_t)(oop base, ptrdiff_t offset, T value); |
104 typedef void (*store_at_func_t)(oop base, ptrdiff_t offset, T value); |
105 typedef T (*atomic_cmpxchg_at_func_t)(T new_value, oop base, ptrdiff_t offset, T compare_value); |
105 typedef T (*atomic_cmpxchg_at_func_t)(T new_value, oop base, ptrdiff_t offset, T compare_value); |
106 typedef T (*atomic_xchg_at_func_t)(T new_value, oop base, ptrdiff_t offset); |
106 typedef T (*atomic_xchg_at_func_t)(oop base, ptrdiff_t offset, T new_value); |
107 |
107 |
108 typedef T (*load_func_t)(void* addr); |
108 typedef T (*load_func_t)(void* addr); |
109 typedef void (*store_func_t)(void* addr, T value); |
109 typedef void (*store_func_t)(void* addr, T value); |
110 typedef T (*atomic_cmpxchg_func_t)(T new_value, void* addr, T compare_value); |
110 typedef T (*atomic_cmpxchg_func_t)(T new_value, void* addr, T compare_value); |
111 typedef T (*atomic_xchg_func_t)(T new_value, void* addr); |
111 typedef T (*atomic_xchg_func_t)(void* addr, T new_value); |
112 |
112 |
113 typedef bool (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, |
113 typedef bool (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, |
114 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, |
114 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, |
115 size_t length); |
115 size_t length); |
116 typedef void (*clone_func_t)(oop src, oop dst, size_t size); |
116 typedef void (*clone_func_t)(oop src, oop dst, size_t size); |
322 atomic_cmpxchg_maybe_locked(T new_value, void* addr, T compare_value); |
322 atomic_cmpxchg_maybe_locked(T new_value, void* addr, T compare_value); |
323 |
323 |
324 template <DecoratorSet ds, typename T> |
324 template <DecoratorSet ds, typename T> |
325 static inline typename EnableIf< |
325 static inline typename EnableIf< |
326 !AccessInternal::PossiblyLockedAccess<T>::value, T>::type |
326 !AccessInternal::PossiblyLockedAccess<T>::value, T>::type |
327 atomic_xchg_maybe_locked(T new_value, void* addr) { |
327 atomic_xchg_maybe_locked(void* addr, T new_value) { |
328 return atomic_xchg_internal<ds>(new_value, addr); |
328 return atomic_xchg_internal<ds>(addr, new_value); |
329 } |
329 } |
330 |
330 |
331 template <DecoratorSet ds, typename T> |
331 template <DecoratorSet ds, typename T> |
332 static typename EnableIf< |
332 static typename EnableIf< |
333 AccessInternal::PossiblyLockedAccess<T>::value, T>::type |
333 AccessInternal::PossiblyLockedAccess<T>::value, T>::type |
334 atomic_xchg_maybe_locked(T new_value, void* addr); |
334 atomic_xchg_maybe_locked(void* addr, T new_value); |
335 |
335 |
336 public: |
336 public: |
337 template <typename T> |
337 template <typename T> |
338 static inline void store(void* addr, T value) { |
338 static inline void store(void* addr, T value) { |
339 store_internal<decorators>(addr, value); |
339 store_internal<decorators>(addr, value); |
348 static inline T atomic_cmpxchg(T new_value, void* addr, T compare_value) { |
348 static inline T atomic_cmpxchg(T new_value, void* addr, T compare_value) { |
349 return atomic_cmpxchg_maybe_locked<decorators>(new_value, addr, compare_value); |
349 return atomic_cmpxchg_maybe_locked<decorators>(new_value, addr, compare_value); |
350 } |
350 } |
351 |
351 |
352 template <typename T> |
352 template <typename T> |
353 static inline T atomic_xchg(T new_value, void* addr) { |
353 static inline T atomic_xchg(void* addr, T new_value) { |
354 return atomic_xchg_maybe_locked<decorators>(new_value, addr); |
354 return atomic_xchg_maybe_locked<decorators>(addr, new_value); |
355 } |
355 } |
356 |
356 |
357 template <typename T> |
357 template <typename T> |
358 static bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, |
358 static bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, |
359 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, |
359 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, |
373 static T oop_atomic_cmpxchg(T new_value, void* addr, T compare_value); |
373 static T oop_atomic_cmpxchg(T new_value, void* addr, T compare_value); |
374 template <typename T> |
374 template <typename T> |
375 static T oop_atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value); |
375 static T oop_atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value); |
376 |
376 |
377 template <typename T> |
377 template <typename T> |
378 static T oop_atomic_xchg(T new_value, void* addr); |
378 static T oop_atomic_xchg(void* addr, T new_value); |
379 template <typename T> |
379 template <typename T> |
380 static T oop_atomic_xchg_at(T new_value, oop base, ptrdiff_t offset); |
380 static T oop_atomic_xchg_at(oop base, ptrdiff_t offset, T new_value); |
381 |
381 |
382 template <typename T> |
382 template <typename T> |
383 static void store_at(oop base, ptrdiff_t offset, T value) { |
383 static void store_at(oop base, ptrdiff_t offset, T value) { |
384 store(field_addr(base, offset), value); |
384 store(field_addr(base, offset), value); |
385 } |
385 } |
393 static T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { |
393 static T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { |
394 return atomic_cmpxchg(new_value, field_addr(base, offset), compare_value); |
394 return atomic_cmpxchg(new_value, field_addr(base, offset), compare_value); |
395 } |
395 } |
396 |
396 |
397 template <typename T> |
397 template <typename T> |
398 static T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { |
398 static T atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) { |
399 return atomic_xchg(new_value, field_addr(base, offset)); |
399 return atomic_xchg(field_addr(base, offset), new_value); |
400 } |
400 } |
401 |
401 |
402 template <typename T> |
402 template <typename T> |
403 static bool oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, |
403 static bool oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, |
404 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, |
404 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, |
537 template <DecoratorSet decorators, typename T> |
537 template <DecoratorSet decorators, typename T> |
538 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>: AllStatic { |
538 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>: AllStatic { |
539 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG>::type func_t; |
539 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG>::type func_t; |
540 static func_t _atomic_xchg_func; |
540 static func_t _atomic_xchg_func; |
541 |
541 |
542 static T atomic_xchg_init(T new_value, void* addr); |
542 static T atomic_xchg_init(void* addr, T new_value); |
543 |
543 |
544 static inline T atomic_xchg(T new_value, void* addr) { |
544 static inline T atomic_xchg(void* addr, T new_value) { |
545 return _atomic_xchg_func(new_value, addr); |
545 return _atomic_xchg_func(addr, new_value); |
546 } |
546 } |
547 }; |
547 }; |
548 |
548 |
549 template <DecoratorSet decorators, typename T> |
549 template <DecoratorSet decorators, typename T> |
550 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>: AllStatic { |
550 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>: AllStatic { |
551 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type func_t; |
551 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type func_t; |
552 static func_t _atomic_xchg_at_func; |
552 static func_t _atomic_xchg_at_func; |
553 |
553 |
554 static T atomic_xchg_at_init(T new_value, oop base, ptrdiff_t offset); |
554 static T atomic_xchg_at_init(oop base, ptrdiff_t offset, T new_value); |
555 |
555 |
556 static inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { |
556 static inline T atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) { |
557 return _atomic_xchg_at_func(new_value, base, offset); |
557 return _atomic_xchg_at_func(base, offset, new_value); |
558 } |
558 } |
559 }; |
559 }; |
560 |
560 |
561 template <DecoratorSet decorators, typename T> |
561 template <DecoratorSet decorators, typename T> |
562 struct RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>: AllStatic { |
562 struct RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>: AllStatic { |
836 } |
836 } |
837 |
837 |
838 template <DecoratorSet decorators, typename T> |
838 template <DecoratorSet decorators, typename T> |
839 inline static typename EnableIf< |
839 inline static typename EnableIf< |
840 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, T>::type |
840 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, T>::type |
841 atomic_xchg(T new_value, void* addr) { |
841 atomic_xchg(void* addr, T new_value) { |
842 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw; |
842 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw; |
843 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) { |
843 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) { |
844 return Raw::oop_atomic_xchg(new_value, addr); |
844 return Raw::oop_atomic_xchg(addr, new_value); |
845 } else { |
845 } else { |
846 return Raw::atomic_xchg(new_value, addr); |
846 return Raw::atomic_xchg(addr, new_value); |
847 } |
847 } |
848 } |
848 } |
849 |
849 |
850 template <DecoratorSet decorators, typename T> |
850 template <DecoratorSet decorators, typename T> |
851 inline static typename EnableIf< |
851 inline static typename EnableIf< |
852 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, T>::type |
852 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, T>::type |
853 atomic_xchg(T new_value, void* addr) { |
853 atomic_xchg(void* addr, T new_value) { |
854 if (UseCompressedOops) { |
854 if (UseCompressedOops) { |
855 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; |
855 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; |
856 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, addr); |
856 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(addr, new_value); |
857 } else { |
857 } else { |
858 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; |
858 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; |
859 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, addr); |
859 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(addr, new_value); |
860 } |
860 } |
861 } |
861 } |
862 |
862 |
863 template <DecoratorSet decorators, typename T> |
863 template <DecoratorSet decorators, typename T> |
864 inline static typename EnableIf< |
864 inline static typename EnableIf< |
865 !HasDecorator<decorators, AS_RAW>::value, T>::type |
865 !HasDecorator<decorators, AS_RAW>::value, T>::type |
866 atomic_xchg(T new_value, void* addr) { |
866 atomic_xchg(void* addr, T new_value) { |
867 if (is_hardwired_primitive<decorators>()) { |
867 if (is_hardwired_primitive<decorators>()) { |
868 const DecoratorSet expanded_decorators = decorators | AS_RAW; |
868 const DecoratorSet expanded_decorators = decorators | AS_RAW; |
869 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, addr); |
869 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(addr, new_value); |
870 } else { |
870 } else { |
871 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>::atomic_xchg(new_value, addr); |
871 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>::atomic_xchg(addr, new_value); |
872 } |
872 } |
873 } |
873 } |
874 |
874 |
875 template <DecoratorSet decorators, typename T> |
875 template <DecoratorSet decorators, typename T> |
876 inline static typename EnableIf< |
876 inline static typename EnableIf< |
877 HasDecorator<decorators, AS_RAW>::value, T>::type |
877 HasDecorator<decorators, AS_RAW>::value, T>::type |
878 atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { |
878 atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) { |
879 return atomic_xchg<decorators>(new_value, field_addr(base, offset)); |
879 return atomic_xchg<decorators>(field_addr(base, offset), new_value); |
880 } |
880 } |
881 |
881 |
882 template <DecoratorSet decorators, typename T> |
882 template <DecoratorSet decorators, typename T> |
883 inline static typename EnableIf< |
883 inline static typename EnableIf< |
884 !HasDecorator<decorators, AS_RAW>::value, T>::type |
884 !HasDecorator<decorators, AS_RAW>::value, T>::type |
885 atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { |
885 atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) { |
886 if (is_hardwired_primitive<decorators>()) { |
886 if (is_hardwired_primitive<decorators>()) { |
887 const DecoratorSet expanded_decorators = decorators | AS_RAW; |
887 const DecoratorSet expanded_decorators = decorators | AS_RAW; |
888 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, base, offset); |
888 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(base, offset, new_value); |
889 } else { |
889 } else { |
890 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::atomic_xchg_at(new_value, base, offset); |
890 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::atomic_xchg_at(base, offset, new_value); |
891 } |
891 } |
892 } |
892 } |
893 |
893 |
894 template <DecoratorSet decorators, typename T> |
894 template <DecoratorSet decorators, typename T> |
895 inline static typename EnableIf< |
895 inline static typename EnableIf< |
1043 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; |
1043 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; |
1044 return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(new_value, addr, compare_value); |
1044 return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(new_value, addr, compare_value); |
1045 } |
1045 } |
1046 |
1046 |
1047 template <DecoratorSet decorators, typename T> |
1047 template <DecoratorSet decorators, typename T> |
1048 inline T atomic_xchg_reduce_types(T new_value, T* addr) { |
1048 inline T atomic_xchg_reduce_types(T* addr, T new_value) { |
1049 const DecoratorSet expanded_decorators = decorators; |
1049 const DecoratorSet expanded_decorators = decorators; |
1050 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, addr); |
1050 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(addr, new_value); |
1051 } |
1051 } |
1052 |
1052 |
1053 template <DecoratorSet decorators> |
1053 template <DecoratorSet decorators> |
1054 inline oop atomic_xchg_reduce_types(oop new_value, narrowOop* addr) { |
1054 inline oop atomic_xchg_reduce_types(narrowOop* addr, oop new_value) { |
1055 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | |
1055 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | |
1056 INTERNAL_RT_USE_COMPRESSED_OOPS; |
1056 INTERNAL_RT_USE_COMPRESSED_OOPS; |
1057 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, addr); |
1057 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(addr, new_value); |
1058 } |
1058 } |
1059 |
1059 |
1060 template <DecoratorSet decorators> |
1060 template <DecoratorSet decorators> |
1061 inline narrowOop atomic_xchg_reduce_types(narrowOop new_value, narrowOop* addr) { |
1061 inline narrowOop atomic_xchg_reduce_types(narrowOop* addr, narrowOop new_value) { |
1062 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | |
1062 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | |
1063 INTERNAL_RT_USE_COMPRESSED_OOPS; |
1063 INTERNAL_RT_USE_COMPRESSED_OOPS; |
1064 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, addr); |
1064 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(addr, new_value); |
1065 } |
1065 } |
1066 |
1066 |
1067 template <DecoratorSet decorators> |
1067 template <DecoratorSet decorators> |
1068 inline oop atomic_xchg_reduce_types(oop new_value, HeapWord* addr) { |
1068 inline oop atomic_xchg_reduce_types(HeapWord* addr, oop new_value) { |
1069 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; |
1069 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; |
1070 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, addr); |
1070 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(addr, new_value); |
1071 } |
1071 } |
1072 |
1072 |
1073 template <DecoratorSet decorators, typename T> |
1073 template <DecoratorSet decorators, typename T> |
1074 inline T load_reduce_types(T* addr) { |
1074 inline T load_reduce_types(T* addr) { |
1075 return PreRuntimeDispatch::load<decorators, T>(addr); |
1075 return PreRuntimeDispatch::load<decorators, T>(addr); |
1222 return PreRuntimeDispatch::atomic_cmpxchg_at<final_decorators>(new_decayed_value, base, |
1222 return PreRuntimeDispatch::atomic_cmpxchg_at<final_decorators>(new_decayed_value, base, |
1223 offset, compare_decayed_value); |
1223 offset, compare_decayed_value); |
1224 } |
1224 } |
1225 |
1225 |
1226 template <DecoratorSet decorators, typename P, typename T> |
1226 template <DecoratorSet decorators, typename P, typename T> |
1227 inline T atomic_xchg(T new_value, P* addr) { |
1227 inline T atomic_xchg(P* addr, T new_value) { |
1228 verify_types<decorators, T>(); |
1228 verify_types<decorators, T>(); |
1229 typedef typename Decay<P>::type DecayedP; |
1229 typedef typename Decay<P>::type DecayedP; |
1230 typedef typename Decay<T>::type DecayedT; |
1230 typedef typename Decay<T>::type DecayedT; |
1231 DecayedT new_decayed_value = new_value; |
1231 DecayedT new_decayed_value = new_value; |
1232 // atomic_xchg is only available in SEQ_CST flavour. |
1232 // atomic_xchg is only available in SEQ_CST flavour. |
1233 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST>::value; |
1233 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST>::value; |
1234 return atomic_xchg_reduce_types<expanded_decorators>(new_decayed_value, |
1234 return atomic_xchg_reduce_types<expanded_decorators>(const_cast<DecayedP*>(addr), |
1235 const_cast<DecayedP*>(addr)); |
1235 new_decayed_value); |
1236 } |
1236 } |
1237 |
1237 |
1238 template <DecoratorSet decorators, typename T> |
1238 template <DecoratorSet decorators, typename T> |
1239 inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { |
1239 inline T atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) { |
1240 verify_types<decorators, T>(); |
1240 verify_types<decorators, T>(); |
1241 typedef typename Decay<T>::type DecayedT; |
1241 typedef typename Decay<T>::type DecayedT; |
1242 DecayedT new_decayed_value = new_value; |
1242 DecayedT new_decayed_value = new_value; |
1243 // atomic_xchg is only available in SEQ_CST flavour. |
1243 // atomic_xchg is only available in SEQ_CST flavour. |
1244 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST | |
1244 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST | |
1245 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ? |
1245 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ? |
1246 INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE)>::value; |
1246 INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE)>::value; |
1247 return PreRuntimeDispatch::atomic_xchg_at<expanded_decorators>(new_decayed_value, base, offset); |
1247 return PreRuntimeDispatch::atomic_xchg_at<expanded_decorators>(base, offset, new_decayed_value); |
1248 } |
1248 } |
1249 |
1249 |
1250 template <DecoratorSet decorators, typename T> |
1250 template <DecoratorSet decorators, typename T> |
1251 inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw, |
1251 inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw, |
1252 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, |
1252 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, |