108 typedef T (*load_func_t)(void* addr); |
108 typedef T (*load_func_t)(void* addr); |
109 typedef void (*store_func_t)(void* addr, T value); |
109 typedef void (*store_func_t)(void* addr, T value); |
110 typedef T (*atomic_cmpxchg_func_t)(T new_value, void* addr, T compare_value); |
110 typedef T (*atomic_cmpxchg_func_t)(T new_value, void* addr, T compare_value); |
111 typedef T (*atomic_xchg_func_t)(T new_value, void* addr); |
111 typedef T (*atomic_xchg_func_t)(T new_value, void* addr); |
112 |
112 |
113 typedef bool (*arraycopy_func_t)(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length); |
113 typedef bool (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, |
|
114 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, |
|
115 size_t length); |
114 typedef void (*clone_func_t)(oop src, oop dst, size_t size); |
116 typedef void (*clone_func_t)(oop src, oop dst, size_t size); |
115 typedef oop (*resolve_func_t)(oop obj); |
117 typedef oop (*resolve_func_t)(oop obj); |
116 typedef bool (*equals_func_t)(oop o1, oop o2); |
118 typedef bool (*equals_func_t)(oop o1, oop o2); |
117 }; |
119 }; |
118 |
120 |
119 template <DecoratorSet decorators> |
121 template <DecoratorSet decorators> |
120 struct AccessFunctionTypes<decorators, void> { |
122 struct AccessFunctionTypes<decorators, void> { |
121 typedef bool (*arraycopy_func_t)(arrayOop src_obj, arrayOop dst_obj, void* src, void* dst, size_t length); |
123 typedef bool (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, void* src, |
|
124 arrayOop dst_obj, size_t dst_offset_in_bytes, void* dst, |
|
125 size_t length); |
122 }; |
126 }; |
123 |
127 |
124 template <DecoratorSet decorators, typename T, BarrierType barrier> struct AccessFunction {}; |
128 template <DecoratorSet decorators, typename T, BarrierType barrier> struct AccessFunction {}; |
125 |
129 |
126 #define ACCESS_GENERATE_ACCESS_FUNCTION(bt, func) \ |
130 #define ACCESS_GENERATE_ACCESS_FUNCTION(bt, func) \ |
351 static inline T atomic_xchg(T new_value, void* addr) { |
355 static inline T atomic_xchg(T new_value, void* addr) { |
352 return atomic_xchg_maybe_locked<decorators>(new_value, addr); |
356 return atomic_xchg_maybe_locked<decorators>(new_value, addr); |
353 } |
357 } |
354 |
358 |
355 template <typename T> |
359 template <typename T> |
356 static bool arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length); |
360 static bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, |
|
361 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, |
|
362 size_t length); |
357 |
363 |
358 template <typename T> |
364 template <typename T> |
359 static void oop_store(void* addr, T value); |
365 static void oop_store(void* addr, T value); |
360 template <typename T> |
366 template <typename T> |
361 static void oop_store_at(oop base, ptrdiff_t offset, T value); |
367 static void oop_store_at(oop base, ptrdiff_t offset, T value); |
557 template <DecoratorSet decorators, typename T> |
565 template <DecoratorSet decorators, typename T> |
558 struct RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>: AllStatic { |
566 struct RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>: AllStatic { |
559 typedef typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type func_t; |
567 typedef typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type func_t; |
560 static func_t _arraycopy_func; |
568 static func_t _arraycopy_func; |
561 |
569 |
562 static bool arraycopy_init(arrayOop src_obj, arrayOop dst_obj, T *src, T* dst, size_t length); |
570 static bool arraycopy_init(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, |
563 |
571 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, |
564 static inline bool arraycopy(arrayOop src_obj, arrayOop dst_obj, T *src, T* dst, size_t length) { |
572 size_t length); |
565 return _arraycopy_func(src_obj, dst_obj, src, dst, length); |
573 |
|
574 static inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, |
|
575 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, |
|
576 size_t length) { |
|
577 return _arraycopy_func(src_obj, src_offset_in_bytes, src_raw, |
|
578 dst_obj, dst_offset_in_bytes, dst_raw, |
|
579 length); |
566 } |
580 } |
567 }; |
581 }; |
568 |
582 |
569 template <DecoratorSet decorators, typename T> |
583 template <DecoratorSet decorators, typename T> |
570 struct RuntimeDispatch<decorators, T, BARRIER_CLONE>: AllStatic { |
584 struct RuntimeDispatch<decorators, T, BARRIER_CLONE>: AllStatic { |
898 } |
912 } |
899 |
913 |
900 template <DecoratorSet decorators, typename T> |
914 template <DecoratorSet decorators, typename T> |
901 inline static typename EnableIf< |
915 inline static typename EnableIf< |
902 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, bool>::type |
916 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, bool>::type |
903 arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { |
917 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, |
|
918 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, |
|
919 size_t length) { |
904 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw; |
920 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw; |
905 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) { |
921 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) { |
906 return Raw::oop_arraycopy(src_obj, dst_obj, src, dst, length); |
922 return Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw, |
907 } else { |
923 dst_obj, dst_offset_in_bytes, dst_raw, |
908 return Raw::arraycopy(src_obj, dst_obj, src, dst, length); |
924 length); |
|
925 } else { |
|
926 return Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw, |
|
927 dst_obj, dst_offset_in_bytes, dst_raw, |
|
928 length); |
909 } |
929 } |
910 } |
930 } |
911 |
931 |
912 template <DecoratorSet decorators, typename T> |
932 template <DecoratorSet decorators, typename T> |
913 inline static typename EnableIf< |
933 inline static typename EnableIf< |
914 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, bool>::type |
934 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, bool>::type |
915 arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { |
935 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, |
|
936 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, |
|
937 size_t length) { |
916 if (UseCompressedOops) { |
938 if (UseCompressedOops) { |
917 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; |
939 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; |
918 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, dst_obj, src, dst, length); |
940 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw, |
|
941 dst_obj, dst_offset_in_bytes, dst_raw, |
|
942 length); |
919 } else { |
943 } else { |
920 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; |
944 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; |
921 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, dst_obj, src, dst, length); |
945 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw, |
|
946 dst_obj, dst_offset_in_bytes, dst_raw, |
|
947 length); |
922 } |
948 } |
923 } |
949 } |
924 |
950 |
925 template <DecoratorSet decorators, typename T> |
951 template <DecoratorSet decorators, typename T> |
926 inline static typename EnableIf< |
952 inline static typename EnableIf< |
927 !HasDecorator<decorators, AS_RAW>::value, bool>::type |
953 !HasDecorator<decorators, AS_RAW>::value, bool>::type |
928 arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { |
954 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, |
|
955 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, |
|
956 size_t length) { |
929 if (is_hardwired_primitive<decorators>()) { |
957 if (is_hardwired_primitive<decorators>()) { |
930 const DecoratorSet expanded_decorators = decorators | AS_RAW; |
958 const DecoratorSet expanded_decorators = decorators | AS_RAW; |
931 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, dst_obj, src, dst, length); |
959 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw, |
932 } else { |
960 dst_obj, dst_offset_in_bytes, dst_raw, |
933 return RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::arraycopy(src_obj, dst_obj, src, dst, length); |
961 length); |
|
962 } else { |
|
963 return RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::arraycopy(src_obj, src_offset_in_bytes, src_raw, |
|
964 dst_obj, dst_offset_in_bytes, dst_raw, |
|
965 length); |
934 } |
966 } |
935 } |
967 } |
936 |
968 |
937 template <DecoratorSet decorators> |
969 template <DecoratorSet decorators> |
938 inline static typename EnableIf< |
970 inline static typename EnableIf< |
1090 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; |
1122 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; |
1091 return PreRuntimeDispatch::load<expanded_decorators, oop>(addr); |
1123 return PreRuntimeDispatch::load<expanded_decorators, oop>(addr); |
1092 } |
1124 } |
1093 |
1125 |
1094 template <DecoratorSet decorators, typename T> |
1126 template <DecoratorSet decorators, typename T> |
1095 inline bool arraycopy_reduce_types(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { |
1127 inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, |
1096 return PreRuntimeDispatch::arraycopy<decorators>(src_obj, dst_obj, src, dst, length); |
1128 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, |
1097 } |
1129 size_t length) { |
1098 |
1130 return PreRuntimeDispatch::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw, |
1099 template <DecoratorSet decorators> |
1131 dst_obj, dst_offset_in_bytes, dst_raw, |
1100 inline bool arraycopy_reduce_types(arrayOop src_obj, arrayOop dst_obj, HeapWord* src, HeapWord* dst, size_t length) { |
1132 length); |
|
1133 } |
|
1134 |
|
1135 template <DecoratorSet decorators> |
|
1136 inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, HeapWord* src_raw, |
|
1137 arrayOop dst_obj, size_t dst_offset_in_bytes, HeapWord* dst_raw, |
|
1138 size_t length) { |
1101 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; |
1139 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; |
1102 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, dst_obj, src, dst, length); |
1140 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw, |
1103 } |
1141 dst_obj, dst_offset_in_bytes, dst_raw, |
1104 |
1142 length); |
1105 template <DecoratorSet decorators> |
1143 } |
1106 inline bool arraycopy_reduce_types(arrayOop src_obj, arrayOop dst_obj, narrowOop* src, narrowOop* dst, size_t length) { |
1144 |
|
1145 template <DecoratorSet decorators> |
|
1146 inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, narrowOop* src_raw, |
|
1147 arrayOop dst_obj, size_t dst_offset_in_bytes, narrowOop* dst_raw, |
|
1148 size_t length) { |
1107 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | |
1149 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | |
1108 INTERNAL_RT_USE_COMPRESSED_OOPS; |
1150 INTERNAL_RT_USE_COMPRESSED_OOPS; |
1109 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, dst_obj, src, dst, length); |
1151 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw, |
|
1152 dst_obj, dst_offset_in_bytes, dst_raw, |
|
1153 length); |
1110 } |
1154 } |
1111 |
1155 |
1112 // Step 1: Set default decorators. This step remembers if a type was volatile |
1156 // Step 1: Set default decorators. This step remembers if a type was volatile |
1113 // and then sets the MO_VOLATILE decorator by default. Otherwise, a default |
1157 // and then sets the MO_VOLATILE decorator by default. Otherwise, a default |
1114 // memory ordering is set for the access, and the implied decorator rules |
1158 // memory ordering is set for the access, and the implied decorator rules |
1237 INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY)>::value; |
1281 INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY)>::value; |
1238 return PreRuntimeDispatch::atomic_xchg_at<expanded_decorators>(new_decayed_value, base, offset); |
1282 return PreRuntimeDispatch::atomic_xchg_at<expanded_decorators>(new_decayed_value, base, offset); |
1239 } |
1283 } |
1240 |
1284 |
1241 template <DecoratorSet decorators, typename T> |
1285 template <DecoratorSet decorators, typename T> |
1242 inline bool arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { |
1286 inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw, |
|
1287 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, |
|
1288 size_t length) { |
1243 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || |
1289 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || |
1244 (IsSame<T, void>::value || IsIntegral<T>::value) || |
1290 (IsSame<T, void>::value || IsIntegral<T>::value) || |
1245 IsFloatingPoint<T>::value)); // arraycopy allows type erased void elements |
1291 IsFloatingPoint<T>::value)); // arraycopy allows type erased void elements |
1246 typedef typename Decay<T>::type DecayedT; |
1292 typedef typename Decay<T>::type DecayedT; |
1247 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IN_HEAP_ARRAY | IN_HEAP>::value; |
1293 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IN_HEAP_ARRAY | IN_HEAP>::value; |
1248 return arraycopy_reduce_types<expanded_decorators>(src_obj, dst_obj, |
1294 return arraycopy_reduce_types<expanded_decorators>(src_obj, src_offset_in_bytes, const_cast<DecayedT*>(src_raw), |
1249 const_cast<DecayedT*>(src), |
1295 dst_obj, dst_offset_in_bytes, const_cast<DecayedT*>(dst_raw), |
1250 const_cast<DecayedT*>(dst), |
|
1251 length); |
1296 length); |
1252 } |
1297 } |
1253 |
1298 |
1254 template <DecoratorSet decorators> |
1299 template <DecoratorSet decorators> |
1255 inline void clone(oop src, oop dst, size_t size) { |
1300 inline void clone(oop src, oop dst, size_t size) { |