8198445: Access API for primitive/native arraycopy
Reviewed-by: pliden, eosterlund, dholmes
--- a/src/hotspot/share/c1/c1_Runtime1.cpp Wed Mar 14 00:30:56 2018 +0900
+++ b/src/hotspot/share/c1/c1_Runtime1.cpp Tue Mar 13 17:13:35 2018 +0100
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2018, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1402,13 +1402,7 @@
Klass* klass_oop = src->klass();
if (klass_oop != dst->klass()) return ac_failed;
TypeArrayKlass* klass = TypeArrayKlass::cast(klass_oop);
- const int l2es = klass->log2_element_size();
- const int ihs = klass->array_header_in_bytes() / wordSize;
- char* src_addr = (char*) ((oopDesc**)src + ihs) + (src_pos << l2es);
- char* dst_addr = (char*) ((oopDesc**)dst + ihs) + (dst_pos << l2es);
- // Potential problem: memmove is not guaranteed to be word atomic
- // Revisit in Merlin
- memmove(dst_addr, src_addr, length << l2es);
+ klass->copy_array(arrayOop(src), src_pos, arrayOop(dst), dst_pos, length, Thread::current());
return ac_ok;
} else if (src->is_objArray() && dst->is_objArray()) {
if (UseCompressedOops) {
--- a/src/hotspot/share/gc/shared/barrierSet.hpp Wed Mar 14 00:30:56 2018 +0900
+++ b/src/hotspot/share/gc/shared/barrierSet.hpp Tue Mar 13 17:13:35 2018 +0100
@@ -189,8 +189,8 @@
}
template <typename T>
- static bool arraycopy_in_heap(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) {
- return Raw::arraycopy(src_obj, dst_obj, src, dst, length);
+ static void arraycopy_in_heap(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) {
+ Raw::arraycopy(src_obj, dst_obj, src, dst, length);
}
// Heap oop accesses. These accessors get resolved when
--- a/src/hotspot/share/oops/access.hpp Wed Mar 14 00:30:56 2018 +0900
+++ b/src/hotspot/share/oops/access.hpp Tue Mar 13 17:13:35 2018 +0100
@@ -402,10 +402,10 @@
}
template <typename T>
- static inline bool arraycopy(arrayOop src_obj, arrayOop dst_obj, T *src, T *dst, size_t length) {
+ static inline void arraycopy(arrayOop src_obj, arrayOop dst_obj, T *src, T *dst, size_t length) {
verify_decorators<ARRAYCOPY_DECORATOR_MASK | IN_HEAP |
AS_DECORATOR_MASK>();
- return AccessInternal::arraycopy<decorators>(src_obj, dst_obj, src, dst, length);
+ AccessInternal::arraycopy<decorators>(src_obj, dst_obj, src, dst, length);
}
// Oop heap accesses
--- a/src/hotspot/share/oops/access.inline.hpp Wed Mar 14 00:30:56 2018 +0900
+++ b/src/hotspot/share/oops/access.inline.hpp Tue Mar 13 17:13:35 2018 +0100
@@ -139,7 +139,8 @@
struct PostRuntimeDispatch<GCBarrierType, BARRIER_ARRAYCOPY, decorators>: public AllStatic {
template <typename T>
static bool access_barrier(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) {
- return GCBarrierType::arraycopy_in_heap(src_obj, dst_obj, src, dst, length);
+ GCBarrierType::arraycopy_in_heap(src_obj, dst_obj, src, dst, length);
+ return true;
}
template <typename T>
@@ -763,7 +764,7 @@
HasDecorator<decorators, AS_RAW>::value, bool>::type
arraycopy(arrayOop src_obj, arrayOop dst_obj, T *src, T* dst, size_t length) {
typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
- return Raw::arraycopy(src, dst, length);
+ return Raw::arraycopy(src_obj, dst_obj, src, dst, length);
}
template <DecoratorSet decorators, typename T>
@@ -1077,7 +1078,9 @@
template <DecoratorSet decorators, typename T>
inline bool arraycopy(arrayOop src_obj, arrayOop dst_obj, T *src, T *dst, size_t length) {
- verify_types<decorators, T>();
+ STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ||
+ (IsSame<T, void>::value || IsIntegral<T>::value) ||
+ IsFloatingPoint<T>::value)); // arraycopy allows type erased void elements
typedef typename Decay<T>::type DecayedT;
const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IN_HEAP_ARRAY | IN_HEAP |
(HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
--- a/src/hotspot/share/oops/accessBackend.cpp Wed Mar 14 00:30:56 2018 +0900
+++ b/src/hotspot/share/oops/accessBackend.cpp Tue Mar 13 17:13:35 2018 +0100
@@ -153,6 +153,13 @@
}
template<>
+ void arraycopy_conjoint<void>(void* src, void* dst, size_t length) {
+ Copy::conjoint_jbytes(reinterpret_cast<jbyte*>(src),
+ reinterpret_cast<jbyte*>(dst),
+ length);
+ }
+
+ template<>
void arraycopy_conjoint_atomic<jbyte>(jbyte* src, jbyte* dst, size_t length) {
Copy::conjoint_jbytes_atomic(src, dst, length);
}
@@ -171,4 +178,9 @@
void arraycopy_conjoint_atomic<jlong>(jlong* src, jlong* dst, size_t length) {
Copy::conjoint_jlongs_atomic(src, dst, length);
}
+
+ template<>
+ void arraycopy_conjoint_atomic<void>(void* src, void* dst, size_t length) {
+ Copy::conjoint_memory_atomic(src, dst, length);
+ }
}
--- a/src/hotspot/share/oops/accessBackend.hpp Wed Mar 14 00:30:56 2018 +0900
+++ b/src/hotspot/share/oops/accessBackend.hpp Tue Mar 13 17:13:35 2018 +0100
@@ -104,6 +104,11 @@
typedef oop (*resolve_func_t)(oop obj);
};
+ template <DecoratorSet decorators>
+ struct AccessFunctionTypes<decorators, void> {
+ typedef bool (*arraycopy_func_t)(arrayOop src_obj, arrayOop dst_obj, void* src, void* dst, size_t length);
+ };
+
template <DecoratorSet decorators, typename T, BarrierType barrier> struct AccessFunction {};
#define ACCESS_GENERATE_ACCESS_FUNCTION(bt, func) \
@@ -335,7 +340,7 @@
}
template <typename T>
- static bool arraycopy(T* src, T* dst, size_t length);
+ static bool arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length);
template <typename T>
static void oop_store(void* addr, T value);
--- a/src/hotspot/share/oops/accessBackend.inline.hpp Wed Mar 14 00:30:56 2018 +0900
+++ b/src/hotspot/share/oops/accessBackend.inline.hpp Tue Mar 13 17:13:35 2018 +0100
@@ -118,7 +118,7 @@
template <DecoratorSet decorators>
template <typename T>
inline bool RawAccessBarrier<decorators>::oop_arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) {
- return arraycopy(src, dst, length);
+ return arraycopy(src_obj, dst_obj, src, dst, length);
}
template <DecoratorSet decorators>
@@ -257,7 +257,7 @@
template <DecoratorSet decorators, typename T>
static inline typename EnableIf<
HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value>::type
- arraycopy(T* src, T* dst, size_t length) {
+ arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) {
// We do not check for ARRAYCOPY_ATOMIC for oops, because they are unconditionally always atomic.
if (HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value) {
AccessInternal::arraycopy_arrayof_conjoint_oops(src, dst, length);
@@ -271,7 +271,7 @@
template <DecoratorSet decorators, typename T>
static inline typename EnableIf<
!HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value>::type
- arraycopy(T* src, T* dst, size_t length) {
+ arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) {
if (HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value) {
AccessInternal::arraycopy_arrayof_conjoint(src, dst, length);
} else if (HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value && sizeof(T) == HeapWordSize) {
@@ -289,12 +289,23 @@
}
}
}
+
+ template <DecoratorSet decorators>
+ static inline typename EnableIf<
+ !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value>::type
+ arraycopy(arrayOop src_obj, arrayOop dst_obj, void* src, void* dst, size_t length) {
+ if (HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value) {
+ AccessInternal::arraycopy_conjoint_atomic(src, dst, length);
+ } else {
+ AccessInternal::arraycopy_conjoint(src, dst, length);
+ }
+ }
};
template <DecoratorSet decorators>
template <typename T>
-inline bool RawAccessBarrier<decorators>::arraycopy(T* src, T* dst, size_t length) {
- RawAccessBarrierArrayCopy::arraycopy<decorators>(src, dst, length);
+inline bool RawAccessBarrier<decorators>::arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) {
+ RawAccessBarrierArrayCopy::arraycopy<decorators>(src_obj, dst_obj, src, dst, length);
return true;
}
--- a/src/hotspot/share/oops/typeArrayKlass.cpp Wed Mar 14 00:30:56 2018 +0900
+++ b/src/hotspot/share/oops/typeArrayKlass.cpp Tue Mar 13 17:13:35 2018 +0100
@@ -152,9 +152,9 @@
// This is an attempt to make the copy_array fast.
int l2es = log2_element_size();
int ihs = array_header_in_bytes() / wordSize;
- char* src = (char*) ((oop*)s + ihs) + ((size_t)src_pos << l2es);
- char* dst = (char*) ((oop*)d + ihs) + ((size_t)dst_pos << l2es);
- Copy::conjoint_memory_atomic(src, dst, (size_t)length << l2es);
+ void* src = (char*) (s->base(element_type())) + ((size_t)src_pos << l2es);
+ void* dst = (char*) (d->base(element_type())) + ((size_t)dst_pos << l2es);
+ HeapAccess<ARRAYCOPY_ATOMIC>::arraycopy(s, d, src, dst, (size_t)length << l2es);
}