/*
* Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef OS_CPU_LINUX_AARCH64_COPY_LINUX_AARCH64_INLINE_HPP
#define OS_CPU_LINUX_AARCH64_COPY_LINUX_AARCH64_INLINE_HPP
#define COPY_SMALL(from, to, count) \
{ \
long tmp0, tmp1, tmp2, tmp3; \
long tmp4, tmp5, tmp6, tmp7; \
__asm volatile( \
" adr %[t0], 0f;" \
" add %[t0], %[t0], %[cnt], lsl #5;" \
" br %[t0];" \
" .align 5;" \
"0:" \
" b 1f;" \
" .align 5;" \
" ldr %[t0], [%[s], #0];" \
" str %[t0], [%[d], #0];" \
" b 1f;" \
" .align 5;" \
" ldp %[t0], %[t1], [%[s], #0];" \
" stp %[t0], %[t1], [%[d], #0];" \
" b 1f;" \
" .align 5;" \
" ldp %[t0], %[t1], [%[s], #0];" \
" ldr %[t2], [%[s], #16];" \
" stp %[t0], %[t1], [%[d], #0];" \
" str %[t2], [%[d], #16];" \
" b 1f;" \
" .align 5;" \
" ldp %[t0], %[t1], [%[s], #0];" \
" ldp %[t2], %[t3], [%[s], #16];" \
" stp %[t0], %[t1], [%[d], #0];" \
" stp %[t2], %[t3], [%[d], #16];" \
" b 1f;" \
" .align 5;" \
" ldp %[t0], %[t1], [%[s], #0];" \
" ldp %[t2], %[t3], [%[s], #16];" \
" ldr %[t4], [%[s], #32];" \
" stp %[t0], %[t1], [%[d], #0];" \
" stp %[t2], %[t3], [%[d], #16];" \
" str %[t4], [%[d], #32];" \
" b 1f;" \
" .align 5;" \
" ldp %[t0], %[t1], [%[s], #0];" \
" ldp %[t2], %[t3], [%[s], #16];" \
" ldp %[t4], %[t5], [%[s], #32];" \
"2:" \
" stp %[t0], %[t1], [%[d], #0];" \
" stp %[t2], %[t3], [%[d], #16];" \
" stp %[t4], %[t5], [%[d], #32];" \
" b 1f;" \
" .align 5;" \
" ldr %[t6], [%[s], #0];" \
" ldp %[t0], %[t1], [%[s], #8];" \
" ldp %[t2], %[t3], [%[s], #24];" \
" ldp %[t4], %[t5], [%[s], #40];" \
" str %[t6], [%[d]], #8;" \
" b 2b;" \
" .align 5;" \
" ldp %[t0], %[t1], [%[s], #0];" \
" ldp %[t2], %[t3], [%[s], #16];" \
" ldp %[t4], %[t5], [%[s], #32];" \
" ldp %[t6], %[t7], [%[s], #48];" \
" stp %[t0], %[t1], [%[d], #0];" \
" stp %[t2], %[t3], [%[d], #16];" \
" stp %[t4], %[t5], [%[d], #32];" \
" stp %[t6], %[t7], [%[d], #48];" \
"1:" \
\
: [s]"+r"(from), [d]"+r"(to), [cnt]"+r"(count), \
[t0]"=&r"(tmp0), [t1]"=&r"(tmp1), [t2]"=&r"(tmp2), [t3]"=&r"(tmp3), \
[t4]"=&r"(tmp4), [t5]"=&r"(tmp5), [t6]"=&r"(tmp6), [t7]"=&r"(tmp7) \
: \
: "memory", "cc"); \
}
static void pd_conjoint_words(const HeapWord* from, HeapWord* to, size_t count) {
__asm volatile( "prfm pldl1strm, [%[s], #0];" :: [s]"r"(from) : "memory");
if (__builtin_expect(count <= 8, 1)) {
COPY_SMALL(from, to, count);
return;
}
_Copy_conjoint_words(from, to, count);
}
static void pd_disjoint_words(const HeapWord* from, HeapWord* to, size_t count) {
if (__builtin_constant_p(count)) {
memcpy(to, from, count * sizeof(HeapWord));
return;
}
__asm volatile( "prfm pldl1strm, [%[s], #0];" :: [s]"r"(from) : "memory");
if (__builtin_expect(count <= 8, 1)) {
COPY_SMALL(from, to, count);
return;
}
_Copy_disjoint_words(from, to, count);
}
static void pd_disjoint_words_atomic(const HeapWord* from, HeapWord* to, size_t count) {
__asm volatile( "prfm pldl1strm, [%[s], #0];" :: [s]"r"(from) : "memory");
if (__builtin_expect(count <= 8, 1)) {
COPY_SMALL(from, to, count);
return;
}
_Copy_disjoint_words(from, to, count);
}
static void pd_aligned_conjoint_words(const HeapWord* from, HeapWord* to, size_t count) {
pd_conjoint_words(from, to, count);
}
static void pd_aligned_disjoint_words(const HeapWord* from, HeapWord* to, size_t count) {
pd_disjoint_words(from, to, count);
}
static void pd_conjoint_bytes(const void* from, void* to, size_t count) {
(void)memmove(to, from, count);
}
static void pd_conjoint_bytes_atomic(const void* from, void* to, size_t count) {
pd_conjoint_bytes(from, to, count);
}
static void pd_conjoint_jshorts_atomic(const jshort* from, jshort* to, size_t count) {
_Copy_conjoint_jshorts_atomic(from, to, count);
}
static void pd_conjoint_jints_atomic(const jint* from, jint* to, size_t count) {
_Copy_conjoint_jints_atomic(from, to, count);
}
static void pd_conjoint_jlongs_atomic(const jlong* from, jlong* to, size_t count) {
_Copy_conjoint_jlongs_atomic(from, to, count);
}
static void pd_conjoint_oops_atomic(const oop* from, oop* to, size_t count) {
assert(BytesPerLong == BytesPerOop, "jlongs and oops must be the same size");
_Copy_conjoint_jlongs_atomic((const jlong*)from, (jlong*)to, count);
}
static void pd_arrayof_conjoint_bytes(const HeapWord* from, HeapWord* to, size_t count) {
_Copy_arrayof_conjoint_bytes(from, to, count);
}
static void pd_arrayof_conjoint_jshorts(const HeapWord* from, HeapWord* to, size_t count) {
_Copy_arrayof_conjoint_jshorts(from, to, count);
}
static void pd_arrayof_conjoint_jints(const HeapWord* from, HeapWord* to, size_t count) {
_Copy_arrayof_conjoint_jints(from, to, count);
}
static void pd_arrayof_conjoint_jlongs(const HeapWord* from, HeapWord* to, size_t count) {
_Copy_arrayof_conjoint_jlongs(from, to, count);
}
static void pd_arrayof_conjoint_oops(const HeapWord* from, HeapWord* to, size_t count) {
assert(!UseCompressedOops, "foo!");
assert(BytesPerLong == BytesPerOop, "jlongs and oops must be the same size");
_Copy_arrayof_conjoint_jlongs(from, to, count);
}
#endif // OS_CPU_LINUX_AARCH64_COPY_LINUX_AARCH64_INLINE_HPP