|
1 # |
|
2 # Copyright (c) 2004, 2013, Oracle and/or its affiliates. All rights reserved. |
|
3 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
|
4 # |
|
5 # This code is free software; you can redistribute it and/or modify it |
|
6 # under the terms of the GNU General Public License version 2 only, as |
|
7 # published by the Free Software Foundation. |
|
8 # |
|
9 # This code is distributed in the hope that it will be useful, but WITHOUT |
|
10 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
|
11 # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
|
12 # version 2 for more details (a copy is included in the LICENSE file that |
|
13 # accompanied this code). |
|
14 # |
|
15 # You should have received a copy of the GNU General Public License version |
|
16 # 2 along with this work; if not, write to the Free Software Foundation, |
|
17 # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
|
18 # |
|
19 # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
|
20 # or visit www.oracle.com if you need additional information or have any |
|
21 # questions. |
|
22 # |
|
23 |
|
24 #ifdef __APPLE__ |
|
25 # Darwin uses _ prefixed global symbols |
|
26 #define SYMBOL(s) _ ## s |
|
27 #define ELF_TYPE(name, description) |
|
28 #else |
|
29 #define SYMBOL(s) s |
|
30 #define ELF_TYPE(name, description) .type name,description |
|
31 #endif |
|
32 |
|
33 # NOTE WELL! The _Copy functions are called directly |
|
34 # from server-compiler-generated code via CallLeafNoFP, |
|
35 # which means that they *must* either not use floating |
|
36 # point or use it in the same manner as does the server |
|
37 # compiler. |
|
38 |
|
39 .globl SYMBOL(_Copy_arrayof_conjoint_bytes) |
|
40 .globl SYMBOL(_Copy_arrayof_conjoint_jshorts) |
|
41 .globl SYMBOL(_Copy_conjoint_jshorts_atomic) |
|
42 .globl SYMBOL(_Copy_arrayof_conjoint_jints) |
|
43 .globl SYMBOL(_Copy_conjoint_jints_atomic) |
|
44 .globl SYMBOL(_Copy_arrayof_conjoint_jlongs) |
|
45 .globl SYMBOL(_Copy_conjoint_jlongs_atomic) |
|
46 |
|
47 .text |
|
48 |
|
49 .globl SYMBOL(SpinPause) |
|
50 .p2align 4,,15 |
|
51 ELF_TYPE(SpinPause,@function) |
|
52 SYMBOL(SpinPause): |
|
53 rep |
|
54 nop |
|
55 movq $1, %rax |
|
56 ret |
|
57 |
|
58 # Support for void Copy::arrayof_conjoint_bytes(void* from, |
|
59 # void* to, |
|
60 # size_t count) |
|
61 # rdi - from |
|
62 # rsi - to |
|
63 # rdx - count, treated as ssize_t |
|
64 # |
|
65 .p2align 4,,15 |
|
66 ELF_TYPE(_Copy_arrayof_conjoint_bytes,@function) |
|
67 SYMBOL(_Copy_arrayof_conjoint_bytes): |
|
68 movq %rdx,%r8 # byte count |
|
69 shrq $3,%rdx # qword count |
|
70 cmpq %rdi,%rsi |
|
71 leaq -1(%rdi,%r8,1),%rax # from + bcount*1 - 1 |
|
72 jbe acb_CopyRight |
|
73 cmpq %rax,%rsi |
|
74 jbe acb_CopyLeft |
|
75 acb_CopyRight: |
|
76 leaq -8(%rdi,%rdx,8),%rax # from + qcount*8 - 8 |
|
77 leaq -8(%rsi,%rdx,8),%rcx # to + qcount*8 - 8 |
|
78 negq %rdx |
|
79 jmp 7f |
|
80 .p2align 4,,15 |
|
81 1: movq 8(%rax,%rdx,8),%rsi |
|
82 movq %rsi,8(%rcx,%rdx,8) |
|
83 addq $1,%rdx |
|
84 jnz 1b |
|
85 2: testq $4,%r8 # check for trailing dword |
|
86 jz 3f |
|
87 movl 8(%rax),%esi # copy trailing dword |
|
88 movl %esi,8(%rcx) |
|
89 addq $4,%rax |
|
90 addq $4,%rcx # original %rsi is trashed, so we |
|
91 # can't use it as a base register |
|
92 3: testq $2,%r8 # check for trailing word |
|
93 jz 4f |
|
94 movw 8(%rax),%si # copy trailing word |
|
95 movw %si,8(%rcx) |
|
96 addq $2,%rcx |
|
97 4: testq $1,%r8 # check for trailing byte |
|
98 jz 5f |
|
99 movb -1(%rdi,%r8,1),%al # copy trailing byte |
|
100 movb %al,8(%rcx) |
|
101 5: ret |
|
102 .p2align 4,,15 |
|
103 6: movq -24(%rax,%rdx,8),%rsi |
|
104 movq %rsi,-24(%rcx,%rdx,8) |
|
105 movq -16(%rax,%rdx,8),%rsi |
|
106 movq %rsi,-16(%rcx,%rdx,8) |
|
107 movq -8(%rax,%rdx,8),%rsi |
|
108 movq %rsi,-8(%rcx,%rdx,8) |
|
109 movq (%rax,%rdx,8),%rsi |
|
110 movq %rsi,(%rcx,%rdx,8) |
|
111 7: addq $4,%rdx |
|
112 jle 6b |
|
113 subq $4,%rdx |
|
114 jl 1b |
|
115 jmp 2b |
|
116 acb_CopyLeft: |
|
117 testq $1,%r8 # check for trailing byte |
|
118 jz 1f |
|
119 movb -1(%rdi,%r8,1),%cl # copy trailing byte |
|
120 movb %cl,-1(%rsi,%r8,1) |
|
121 subq $1,%r8 # adjust for possible trailing word |
|
122 1: testq $2,%r8 # check for trailing word |
|
123 jz 2f |
|
124 movw -2(%rdi,%r8,1),%cx # copy trailing word |
|
125 movw %cx,-2(%rsi,%r8,1) |
|
126 2: testq $4,%r8 # check for trailing dword |
|
127 jz 5f |
|
128 movl (%rdi,%rdx,8),%ecx # copy trailing dword |
|
129 movl %ecx,(%rsi,%rdx,8) |
|
130 jmp 5f |
|
131 .p2align 4,,15 |
|
132 3: movq -8(%rdi,%rdx,8),%rcx |
|
133 movq %rcx,-8(%rsi,%rdx,8) |
|
134 subq $1,%rdx |
|
135 jnz 3b |
|
136 ret |
|
137 .p2align 4,,15 |
|
138 4: movq 24(%rdi,%rdx,8),%rcx |
|
139 movq %rcx,24(%rsi,%rdx,8) |
|
140 movq 16(%rdi,%rdx,8),%rcx |
|
141 movq %rcx,16(%rsi,%rdx,8) |
|
142 movq 8(%rdi,%rdx,8),%rcx |
|
143 movq %rcx,8(%rsi,%rdx,8) |
|
144 movq (%rdi,%rdx,8),%rcx |
|
145 movq %rcx,(%rsi,%rdx,8) |
|
146 5: subq $4,%rdx |
|
147 jge 4b |
|
148 addq $4,%rdx |
|
149 jg 3b |
|
150 ret |
|
151 |
|
152 # Support for void Copy::arrayof_conjoint_jshorts(void* from, |
|
153 # void* to, |
|
154 # size_t count) |
|
155 # Equivalent to |
|
156 # conjoint_jshorts_atomic |
|
157 # |
|
158 # If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we |
|
159 # let the hardware handle it. The tow or four words within dwords |
|
160 # or qwords that span cache line boundaries will still be loaded |
|
161 # and stored atomically. |
|
162 # |
|
163 # rdi - from |
|
164 # rsi - to |
|
165 # rdx - count, treated as ssize_t |
|
166 # |
|
167 .p2align 4,,15 |
|
168 ELF_TYPE(_Copy_arrayof_conjoint_jshorts,@function) |
|
169 ELF_TYPE(_Copy_conjoint_jshorts_atomic,@function) |
|
170 SYMBOL(_Copy_arrayof_conjoint_jshorts): |
|
171 SYMBOL(_Copy_conjoint_jshorts_atomic): |
|
172 movq %rdx,%r8 # word count |
|
173 shrq $2,%rdx # qword count |
|
174 cmpq %rdi,%rsi |
|
175 leaq -2(%rdi,%r8,2),%rax # from + wcount*2 - 2 |
|
176 jbe acs_CopyRight |
|
177 cmpq %rax,%rsi |
|
178 jbe acs_CopyLeft |
|
179 acs_CopyRight: |
|
180 leaq -8(%rdi,%rdx,8),%rax # from + qcount*8 - 8 |
|
181 leaq -8(%rsi,%rdx,8),%rcx # to + qcount*8 - 8 |
|
182 negq %rdx |
|
183 jmp 6f |
|
184 1: movq 8(%rax,%rdx,8),%rsi |
|
185 movq %rsi,8(%rcx,%rdx,8) |
|
186 addq $1,%rdx |
|
187 jnz 1b |
|
188 2: testq $2,%r8 # check for trailing dword |
|
189 jz 3f |
|
190 movl 8(%rax),%esi # copy trailing dword |
|
191 movl %esi,8(%rcx) |
|
192 addq $4,%rcx # original %rsi is trashed, so we |
|
193 # can't use it as a base register |
|
194 3: testq $1,%r8 # check for trailing word |
|
195 jz 4f |
|
196 movw -2(%rdi,%r8,2),%si # copy trailing word |
|
197 movw %si,8(%rcx) |
|
198 4: ret |
|
199 .p2align 4,,15 |
|
200 5: movq -24(%rax,%rdx,8),%rsi |
|
201 movq %rsi,-24(%rcx,%rdx,8) |
|
202 movq -16(%rax,%rdx,8),%rsi |
|
203 movq %rsi,-16(%rcx,%rdx,8) |
|
204 movq -8(%rax,%rdx,8),%rsi |
|
205 movq %rsi,-8(%rcx,%rdx,8) |
|
206 movq (%rax,%rdx,8),%rsi |
|
207 movq %rsi,(%rcx,%rdx,8) |
|
208 6: addq $4,%rdx |
|
209 jle 5b |
|
210 subq $4,%rdx |
|
211 jl 1b |
|
212 jmp 2b |
|
213 acs_CopyLeft: |
|
214 testq $1,%r8 # check for trailing word |
|
215 jz 1f |
|
216 movw -2(%rdi,%r8,2),%cx # copy trailing word |
|
217 movw %cx,-2(%rsi,%r8,2) |
|
218 1: testq $2,%r8 # check for trailing dword |
|
219 jz 4f |
|
220 movl (%rdi,%rdx,8),%ecx # copy trailing dword |
|
221 movl %ecx,(%rsi,%rdx,8) |
|
222 jmp 4f |
|
223 2: movq -8(%rdi,%rdx,8),%rcx |
|
224 movq %rcx,-8(%rsi,%rdx,8) |
|
225 subq $1,%rdx |
|
226 jnz 2b |
|
227 ret |
|
228 .p2align 4,,15 |
|
229 3: movq 24(%rdi,%rdx,8),%rcx |
|
230 movq %rcx,24(%rsi,%rdx,8) |
|
231 movq 16(%rdi,%rdx,8),%rcx |
|
232 movq %rcx,16(%rsi,%rdx,8) |
|
233 movq 8(%rdi,%rdx,8),%rcx |
|
234 movq %rcx,8(%rsi,%rdx,8) |
|
235 movq (%rdi,%rdx,8),%rcx |
|
236 movq %rcx,(%rsi,%rdx,8) |
|
237 4: subq $4,%rdx |
|
238 jge 3b |
|
239 addq $4,%rdx |
|
240 jg 2b |
|
241 ret |
|
242 |
|
243 # Support for void Copy::arrayof_conjoint_jints(jint* from, |
|
244 # jint* to, |
|
245 # size_t count) |
|
246 # Equivalent to |
|
247 # conjoint_jints_atomic |
|
248 # |
|
249 # If 'from' and/or 'to' are aligned on 4-byte boundaries, we let |
|
250 # the hardware handle it. The two dwords within qwords that span |
|
251 # cache line boundaries will still be loaded and stored atomically. |
|
252 # |
|
253 # rdi - from |
|
254 # rsi - to |
|
255 # rdx - count, treated as ssize_t |
|
256 # |
|
257 .p2align 4,,15 |
|
258 ELF_TYPE(_Copy_arrayof_conjoint_jints,@function) |
|
259 ELF_TYPE(_Copy_conjoint_jints_atomic,@function) |
|
260 SYMBOL(_Copy_arrayof_conjoint_jints): |
|
261 SYMBOL(_Copy_conjoint_jints_atomic): |
|
262 movq %rdx,%r8 # dword count |
|
263 shrq %rdx # qword count |
|
264 cmpq %rdi,%rsi |
|
265 leaq -4(%rdi,%r8,4),%rax # from + dcount*4 - 4 |
|
266 jbe aci_CopyRight |
|
267 cmpq %rax,%rsi |
|
268 jbe aci_CopyLeft |
|
269 aci_CopyRight: |
|
270 leaq -8(%rdi,%rdx,8),%rax # from + qcount*8 - 8 |
|
271 leaq -8(%rsi,%rdx,8),%rcx # to + qcount*8 - 8 |
|
272 negq %rdx |
|
273 jmp 5f |
|
274 .p2align 4,,15 |
|
275 1: movq 8(%rax,%rdx,8),%rsi |
|
276 movq %rsi,8(%rcx,%rdx,8) |
|
277 addq $1,%rdx |
|
278 jnz 1b |
|
279 2: testq $1,%r8 # check for trailing dword |
|
280 jz 3f |
|
281 movl 8(%rax),%esi # copy trailing dword |
|
282 movl %esi,8(%rcx) |
|
283 3: ret |
|
284 .p2align 4,,15 |
|
285 4: movq -24(%rax,%rdx,8),%rsi |
|
286 movq %rsi,-24(%rcx,%rdx,8) |
|
287 movq -16(%rax,%rdx,8),%rsi |
|
288 movq %rsi,-16(%rcx,%rdx,8) |
|
289 movq -8(%rax,%rdx,8),%rsi |
|
290 movq %rsi,-8(%rcx,%rdx,8) |
|
291 movq (%rax,%rdx,8),%rsi |
|
292 movq %rsi,(%rcx,%rdx,8) |
|
293 5: addq $4,%rdx |
|
294 jle 4b |
|
295 subq $4,%rdx |
|
296 jl 1b |
|
297 jmp 2b |
|
298 aci_CopyLeft: |
|
299 testq $1,%r8 # check for trailing dword |
|
300 jz 3f |
|
301 movl -4(%rdi,%r8,4),%ecx # copy trailing dword |
|
302 movl %ecx,-4(%rsi,%r8,4) |
|
303 jmp 3f |
|
304 1: movq -8(%rdi,%rdx,8),%rcx |
|
305 movq %rcx,-8(%rsi,%rdx,8) |
|
306 subq $1,%rdx |
|
307 jnz 1b |
|
308 ret |
|
309 .p2align 4,,15 |
|
310 2: movq 24(%rdi,%rdx,8),%rcx |
|
311 movq %rcx,24(%rsi,%rdx,8) |
|
312 movq 16(%rdi,%rdx,8),%rcx |
|
313 movq %rcx,16(%rsi,%rdx,8) |
|
314 movq 8(%rdi,%rdx,8),%rcx |
|
315 movq %rcx,8(%rsi,%rdx,8) |
|
316 movq (%rdi,%rdx,8),%rcx |
|
317 movq %rcx,(%rsi,%rdx,8) |
|
318 3: subq $4,%rdx |
|
319 jge 2b |
|
320 addq $4,%rdx |
|
321 jg 1b |
|
322 ret |
|
323 |
|
324 # Support for void Copy::arrayof_conjoint_jlongs(jlong* from, |
|
325 # jlong* to, |
|
326 # size_t count) |
|
327 # Equivalent to |
|
328 # conjoint_jlongs_atomic |
|
329 # arrayof_conjoint_oops |
|
330 # conjoint_oops_atomic |
|
331 # |
|
332 # rdi - from |
|
333 # rsi - to |
|
334 # rdx - count, treated as ssize_t |
|
335 # |
|
336 .p2align 4,,15 |
|
337 ELF_TYPE(_Copy_arrayof_conjoint_jlongs,@function) |
|
338 ELF_TYPE(_Copy_conjoint_jlongs_atomic,@function) |
|
339 SYMBOL(_Copy_arrayof_conjoint_jlongs): |
|
340 SYMBOL(_Copy_conjoint_jlongs_atomic): |
|
341 cmpq %rdi,%rsi |
|
342 leaq -8(%rdi,%rdx,8),%rax # from + count*8 - 8 |
|
343 jbe acl_CopyRight |
|
344 cmpq %rax,%rsi |
|
345 jbe acl_CopyLeft |
|
346 acl_CopyRight: |
|
347 leaq -8(%rsi,%rdx,8),%rcx # to + count*8 - 8 |
|
348 negq %rdx |
|
349 jmp 3f |
|
350 1: movq 8(%rax,%rdx,8),%rsi |
|
351 movq %rsi,8(%rcx,%rdx,8) |
|
352 addq $1,%rdx |
|
353 jnz 1b |
|
354 ret |
|
355 .p2align 4,,15 |
|
356 2: movq -24(%rax,%rdx,8),%rsi |
|
357 movq %rsi,-24(%rcx,%rdx,8) |
|
358 movq -16(%rax,%rdx,8),%rsi |
|
359 movq %rsi,-16(%rcx,%rdx,8) |
|
360 movq -8(%rax,%rdx,8),%rsi |
|
361 movq %rsi,-8(%rcx,%rdx,8) |
|
362 movq (%rax,%rdx,8),%rsi |
|
363 movq %rsi,(%rcx,%rdx,8) |
|
364 3: addq $4,%rdx |
|
365 jle 2b |
|
366 subq $4,%rdx |
|
367 jl 1b |
|
368 ret |
|
369 4: movq -8(%rdi,%rdx,8),%rcx |
|
370 movq %rcx,-8(%rsi,%rdx,8) |
|
371 subq $1,%rdx |
|
372 jnz 4b |
|
373 ret |
|
374 .p2align 4,,15 |
|
375 5: movq 24(%rdi,%rdx,8),%rcx |
|
376 movq %rcx,24(%rsi,%rdx,8) |
|
377 movq 16(%rdi,%rdx,8),%rcx |
|
378 movq %rcx,16(%rsi,%rdx,8) |
|
379 movq 8(%rdi,%rdx,8),%rcx |
|
380 movq %rcx,8(%rsi,%rdx,8) |
|
381 movq (%rdi,%rdx,8),%rcx |
|
382 movq %rcx,(%rsi,%rdx,8) |
|
383 acl_CopyLeft: |
|
384 subq $4,%rdx |
|
385 jge 5b |
|
386 addq $4,%rdx |
|
387 jg 4b |
|
388 ret |