/
/ Copyright (c) 2004, 2013, Oracle and/or its affiliates. All rights reserved.
/ DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
/
/ This code is free software; you can redistribute it and/or modify it
/ under the terms of the GNU General Public License version 2 only, as
/ published by the Free Software Foundation.
/
/ This code is distributed in the hope that it will be useful, but WITHOUT
/ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
/ FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
/ version 2 for more details (a copy is included in the LICENSE file that
/ accompanied this code).
/
/ You should have received a copy of the GNU General Public License version
/ 2 along with this work; if not, write to the Free Software Foundation,
/ Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
/
/ Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
/ or visit www.oracle.com if you need additional information or have any
/ questions.
/

        .globl fs_load
        .globl fs_thread

        // NOTE WELL!  The _Copy functions are called directly
        // from server-compiler-generated code via CallLeafNoFP,
        // which means that they *must* either not use floating
        // point or use it in the same manner as does the server
        // compiler.

        .globl _Copy_arrayof_conjoint_bytes
        .globl _Copy_conjoint_jshorts_atomic
        .globl _Copy_arrayof_conjoint_jshorts
        .globl _Copy_conjoint_jints_atomic
        .globl _Copy_arrayof_conjoint_jints
        .globl _Copy_conjoint_jlongs_atomic
        .globl _Copy_arrayof_conjoint_jlongs

        .section .text,"ax"

        / Fast thread accessors, used by threadLS_solaris_amd64.cpp
        .align   16
fs_load:
        movq %fs:(%rdi),%rax
        ret

        .align   16
fs_thread:
        movq %fs:0x0,%rax
        ret

        .globl  SpinPause
        .align  16
SpinPause:
        rep
        nop
        movq    $1, %rax
        ret


        / Support for void Copy::arrayof_conjoint_bytes(void* from,
        /                                               void* to,
        /                                               size_t count)
        / rdi - from
        / rsi - to
        / rdx - count, treated as ssize_t
        /
        .align   16
_Copy_arrayof_conjoint_bytes:
        movq     %rdx,%r8             / byte count
        shrq     $3,%rdx              / qword count
        cmpq     %rdi,%rsi
        leaq     -1(%rdi,%r8,1),%rax  / from + bcount*1 - 1
        jbe      acb_CopyRight
        cmpq     %rax,%rsi
        jbe      acb_CopyLeft 
acb_CopyRight:
        leaq     -8(%rdi,%rdx,8),%rax / from + qcount*8 - 8
        leaq     -8(%rsi,%rdx,8),%rcx / to + qcount*8 - 8
        negq     %rdx
        jmp      7f
        .align   16
1:      movq     8(%rax,%rdx,8),%rsi
        movq     %rsi,8(%rcx,%rdx,8)
        addq     $1,%rdx
        jnz      1b
2:      testq    $4,%r8               / check for trailing dword
        jz       3f
        movl     8(%rax),%esi         / copy trailing dword
        movl     %esi,8(%rcx)
        addq     $4,%rax
        addq     $4,%rcx              / original %rsi is trashed, so we
                                      /  can't use it as a base register
3:      testq    $2,%r8               / check for trailing word
        jz       4f
        movw     8(%rax),%si          / copy trailing word
        movw     %si,8(%rcx)
        addq     $2,%rcx
4:      testq    $1,%r8               / check for trailing byte
        jz       5f
        movb     -1(%rdi,%r8,1),%al   / copy trailing byte
        movb     %al,8(%rcx)
5:      ret
        .align   16
6:      movq     -24(%rax,%rdx,8),%rsi
        movq     %rsi,-24(%rcx,%rdx,8)
        movq     -16(%rax,%rdx,8),%rsi
        movq     %rsi,-16(%rcx,%rdx,8)
        movq     -8(%rax,%rdx,8),%rsi
        movq     %rsi,-8(%rcx,%rdx,8)
        movq     (%rax,%rdx,8),%rsi
        movq     %rsi,(%rcx,%rdx,8)
7:      addq     $4,%rdx
        jle      6b
        subq     $4,%rdx
        jl       1b
        jmp      2b
acb_CopyLeft:
        testq    $1,%r8               / check for trailing byte
        jz       1f
        movb     -1(%rdi,%r8,1),%cl   / copy trailing byte
        movb     %cl,-1(%rsi,%r8,1)
        subq     $1,%r8               / adjust for possible trailing word
1:      testq    $2,%r8               / check for trailing word
        jz       2f
        movw     -2(%rdi,%r8,1),%cx   / copy trailing word
        movw     %cx,-2(%rsi,%r8,1)
2:      testq    $4,%r8               / check for trailing dword
        jz       5f
        movl     (%rdi,%rdx,8),%ecx   / copy trailing dword
        movl     %ecx,(%rsi,%rdx,8)
        jmp      5f
        .align   16
3:      movq     -8(%rdi,%rdx,8),%rcx
        movq     %rcx,-8(%rsi,%rdx,8)
        subq     $1,%rdx
        jnz      3b
        ret
        .align   16
4:      movq     24(%rdi,%rdx,8),%rcx
        movq     %rcx,24(%rsi,%rdx,8)
        movq     16(%rdi,%rdx,8),%rcx
        movq     %rcx,16(%rsi,%rdx,8)
        movq     8(%rdi,%rdx,8),%rcx
        movq     %rcx,8(%rsi,%rdx,8)
        movq     (%rdi,%rdx,8),%rcx
        movq     %rcx,(%rsi,%rdx,8)
5:      subq     $4,%rdx
        jge      4b
        addq     $4,%rdx
        jg       3b
        ret

        / Support for void Copy::arrayof_conjoint_jshorts(void* from,
        /                                                 void* to,
        /                                                 size_t count)
        / Equivalent to
        /   conjoint_jshorts_atomic
        /
        / If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we
        / let the hardware handle it.  The tow or four words within dwords
        / or qwords that span cache line boundaries will still be loaded
        / and stored atomically.
        /
        / rdi - from
        / rsi - to
        / rdx - count, treated as ssize_t
        /
        .align   16
_Copy_arrayof_conjoint_jshorts:
_Copy_conjoint_jshorts_atomic:
        movq     %rdx,%r8             / word count
        shrq     $2,%rdx              / qword count
        cmpq     %rdi,%rsi
        leaq     -2(%rdi,%r8,2),%rax  / from + wcount*2 - 2
        jbe      acs_CopyRight
        cmpq     %rax,%rsi
        jbe      acs_CopyLeft 
acs_CopyRight:
        leaq     -8(%rdi,%rdx,8),%rax / from + qcount*8 - 8
        leaq     -8(%rsi,%rdx,8),%rcx / to + qcount*8 - 8
        negq     %rdx
        jmp      6f
1:      movq     8(%rax,%rdx,8),%rsi
        movq     %rsi,8(%rcx,%rdx,8)
        addq     $1,%rdx
        jnz      1b
2:      testq    $2,%r8               / check for trailing dword
        jz       3f
        movl     8(%rax),%esi         / copy trailing dword
        movl     %esi,8(%rcx)
        addq     $4,%rcx              / original %rsi is trashed, so we
                                      /  can't use it as a base register
3:      testq    $1,%r8               / check for trailing word
        jz       4f
        movw     -2(%rdi,%r8,2),%si   / copy trailing word
        movw     %si,8(%rcx)
4:      ret
        .align   16
5:      movq     -24(%rax,%rdx,8),%rsi
        movq     %rsi,-24(%rcx,%rdx,8)
        movq     -16(%rax,%rdx,8),%rsi
        movq     %rsi,-16(%rcx,%rdx,8)
        movq     -8(%rax,%rdx,8),%rsi
        movq     %rsi,-8(%rcx,%rdx,8)
        movq     (%rax,%rdx,8),%rsi
        movq     %rsi,(%rcx,%rdx,8)
6:      addq     $4,%rdx
        jle      5b
        subq     $4,%rdx
        jl       1b
        jmp      2b
acs_CopyLeft:
        testq    $1,%r8               / check for trailing word
        jz       1f
        movw     -2(%rdi,%r8,2),%cx   / copy trailing word
        movw     %cx,-2(%rsi,%r8,2)
1:      testq    $2,%r8               / check for trailing dword
        jz       4f
        movl     (%rdi,%rdx,8),%ecx   / copy trailing dword
        movl     %ecx,(%rsi,%rdx,8)
        jmp      4f
2:      movq     -8(%rdi,%rdx,8),%rcx
        movq     %rcx,-8(%rsi,%rdx,8)
        subq     $1,%rdx
        jnz      2b
        ret
        .align   16
3:      movq     24(%rdi,%rdx,8),%rcx
        movq     %rcx,24(%rsi,%rdx,8)
        movq     16(%rdi,%rdx,8),%rcx
        movq     %rcx,16(%rsi,%rdx,8)
        movq     8(%rdi,%rdx,8),%rcx
        movq     %rcx,8(%rsi,%rdx,8)
        movq     (%rdi,%rdx,8),%rcx
        movq     %rcx,(%rsi,%rdx,8)
4:      subq     $4,%rdx
        jge      3b
        addq     $4,%rdx
        jg       2b
        ret

        / Support for void Copy::arrayof_conjoint_jints(jint* from,
        /                                               jint* to,
        /                                               size_t count)
        / Equivalent to
        /   conjoint_jints_atomic
        /
        / If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
        / the hardware handle it.  The two dwords within qwords that span
        / cache line boundaries will still be loaded and stored atomically.
        /
        / rdi - from
        / rsi - to
        / rdx - count, treated as ssize_t
        /
        .align   16
_Copy_arrayof_conjoint_jints:
_Copy_conjoint_jints_atomic:
        movq     %rdx,%r8             / dword count
        shrq     %rdx                 / qword count
        cmpq     %rdi,%rsi
        leaq     -4(%rdi,%r8,4),%rax  / from + dcount*4 - 4
        jbe      aci_CopyRight
        cmpq     %rax,%rsi
        jbe      aci_CopyLeft 
aci_CopyRight:
        leaq     -8(%rdi,%rdx,8),%rax / from + qcount*8 - 8
        leaq     -8(%rsi,%rdx,8),%rcx / to + qcount*8 - 8
        negq     %rdx
        jmp      5f
        .align   16
1:      movq     8(%rax,%rdx,8),%rsi
        movq     %rsi,8(%rcx,%rdx,8)
        addq     $1,%rdx
        jnz       1b
2:      testq    $1,%r8               / check for trailing dword
        jz       3f
        movl     8(%rax),%esi         / copy trailing dword
        movl     %esi,8(%rcx)
3:      ret
        .align   16
4:      movq     -24(%rax,%rdx,8),%rsi
        movq     %rsi,-24(%rcx,%rdx,8)
        movq     -16(%rax,%rdx,8),%rsi
        movq     %rsi,-16(%rcx,%rdx,8)
        movq     -8(%rax,%rdx,8),%rsi
        movq     %rsi,-8(%rcx,%rdx,8)
        movq     (%rax,%rdx,8),%rsi
        movq     %rsi,(%rcx,%rdx,8)
5:      addq     $4,%rdx
        jle      4b
        subq     $4,%rdx
        jl       1b
        jmp      2b
aci_CopyLeft:
        testq    $1,%r8               / check for trailing dword
        jz       3f
        movl     -4(%rdi,%r8,4),%ecx  / copy trailing dword
        movl     %ecx,-4(%rsi,%r8,4)
        jmp      3f
1:      movq     -8(%rdi,%rdx,8),%rcx
        movq     %rcx,-8(%rsi,%rdx,8)
        subq     $1,%rdx
        jnz      1b
        ret
        .align   16
2:      movq     24(%rdi,%rdx,8),%rcx
        movq     %rcx,24(%rsi,%rdx,8)
        movq     16(%rdi,%rdx,8),%rcx
        movq     %rcx,16(%rsi,%rdx,8)
        movq     8(%rdi,%rdx,8),%rcx
        movq     %rcx,8(%rsi,%rdx,8)
        movq     (%rdi,%rdx,8),%rcx
        movq     %rcx,(%rsi,%rdx,8)
3:      subq     $4,%rdx
        jge      2b
        addq     $4,%rdx
        jg       1b
        ret

        / Support for void Copy::arrayof_conjoint_jlongs(jlong* from,
        /                                                jlong* to,
        /                                                size_t count)
        / Equivalent to
        /   conjoint_jlongs_atomic
        /   arrayof_conjoint_oops
        /   conjoint_oops_atomic
        /
        / rdi - from
        / rsi - to
        / rdx - count, treated as ssize_t
        /
        .align   16
_Copy_arrayof_conjoint_jlongs:
_Copy_conjoint_jlongs_atomic:
        cmpq     %rdi,%rsi
        leaq     -8(%rdi,%rdx,8),%rax / from + count*8 - 8
        jbe      acl_CopyRight
        cmpq     %rax,%rsi
        jbe      acl_CopyLeft 
acl_CopyRight:
        leaq     -8(%rsi,%rdx,8),%rcx / to + count*8 - 8
        negq     %rdx
        jmp      3f
1:      movq     8(%rax,%rdx,8),%rsi
        movq     %rsi,8(%rcx,%rdx,8)
        addq     $1,%rdx
        jnz      1b
        ret
        .align   16
2:      movq     -24(%rax,%rdx,8),%rsi
        movq     %rsi,-24(%rcx,%rdx,8)
        movq     -16(%rax,%rdx,8),%rsi
        movq     %rsi,-16(%rcx,%rdx,8)
        movq     -8(%rax,%rdx,8),%rsi
        movq     %rsi,-8(%rcx,%rdx,8)
        movq     (%rax,%rdx,8),%rsi
        movq     %rsi,(%rcx,%rdx,8)
3:      addq     $4,%rdx
        jle      2b
        subq     $4,%rdx
        jl       1b
        ret
4:      movq     -8(%rdi,%rdx,8),%rcx
        movq     %rcx,-8(%rsi,%rdx,8)
        subq     $1,%rdx
        jnz      4b
        ret
        .align   16
5:      movq     24(%rdi,%rdx,8),%rcx
        movq     %rcx,24(%rsi,%rdx,8)
        movq     16(%rdi,%rdx,8),%rcx
        movq     %rcx,16(%rsi,%rdx,8)
        movq     8(%rdi,%rdx,8),%rcx
        movq     %rcx,8(%rsi,%rdx,8)
        movq     (%rdi,%rdx,8),%rcx
        movq     %rcx,(%rsi,%rdx,8)
acl_CopyLeft:
        subq     $4,%rdx
        jge      5b
        addq     $4,%rdx
        jg       4b
        ret