author | stefank |
Mon, 25 Nov 2019 12:31:39 +0100 | |
changeset 59249 | 29b0d0b61615 |
parent 55490 | 3f3dc00a69a5 |
child 59251 | 4cbfa5077d68 |
permissions | -rw-r--r-- |
42664 | 1 |
/* |
55490
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
2 |
* Copyright (c) 2008, 2019, Oracle and/or its affiliates. All rights reserved. |
42664 | 3 |
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
4 |
* |
|
5 |
* This code is free software; you can redistribute it and/or modify it |
|
6 |
* under the terms of the GNU General Public License version 2 only, as |
|
7 |
* published by the Free Software Foundation. |
|
8 |
* |
|
9 |
* This code is distributed in the hope that it will be useful, but WITHOUT |
|
10 |
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
|
11 |
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
|
12 |
* version 2 for more details (a copy is included in the LICENSE file that |
|
13 |
* accompanied this code). |
|
14 |
* |
|
15 |
* You should have received a copy of the GNU General Public License version |
|
16 |
* 2 along with this work; if not, write to the Free Software Foundation, |
|
17 |
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
|
18 |
* |
|
19 |
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
|
20 |
* or visit www.oracle.com if you need additional information or have any |
|
21 |
* questions. |
|
22 |
* |
|
23 |
*/ |
|
24 |
||
25 |
#include "precompiled.hpp" |
|
26 |
#include "asm/assembler.hpp" |
|
27 |
#include "assembler_arm.inline.hpp" |
|
49484
ee8fa73b90f9
8198949: Modularize arraycopy stub routine GC barriers
eosterlund
parents:
49455
diff
changeset
|
28 |
#include "gc/shared/barrierSet.hpp" |
ee8fa73b90f9
8198949: Modularize arraycopy stub routine GC barriers
eosterlund
parents:
49455
diff
changeset
|
29 |
#include "gc/shared/barrierSetAssembler.hpp" |
42664 | 30 |
#include "interpreter/interpreter.hpp" |
54786 | 31 |
#include "memory/universe.hpp" |
42664 | 32 |
#include "nativeInst_arm.hpp" |
33 |
#include "oops/instanceOop.hpp" |
|
34 |
#include "oops/method.hpp" |
|
35 |
#include "oops/objArrayKlass.hpp" |
|
36 |
#include "oops/oop.inline.hpp" |
|
37 |
#include "prims/methodHandles.hpp" |
|
38 |
#include "runtime/frame.inline.hpp" |
|
39 |
#include "runtime/handles.inline.hpp" |
|
40 |
#include "runtime/sharedRuntime.hpp" |
|
41 |
#include "runtime/stubCodeGenerator.hpp" |
|
42 |
#include "runtime/stubRoutines.hpp" |
|
46625 | 43 |
#include "utilities/align.hpp" |
42664 | 44 |
#ifdef COMPILER2 |
45 |
#include "opto/runtime.hpp" |
|
46 |
#endif |
|
47 |
||
48 |
// Declaration and definition of StubGenerator (no .hpp file). |
|
49 |
// For a more detailed description of the stub routine structure |
|
50 |
// see the comment in stubRoutines.hpp |
|
51 |
||
52 |
#define __ _masm-> |
|
53 |
||
54 |
#ifdef PRODUCT |
|
55 |
#define BLOCK_COMMENT(str) /* nothing */ |
|
56 |
#else |
|
57 |
#define BLOCK_COMMENT(str) __ block_comment(str) |
|
58 |
#endif |
|
59 |
||
60 |
#define BIND(label) bind(label); BLOCK_COMMENT(#label ":") |
|
61 |
||
62 |
// ------------------------------------------------------------------------------------------------------------------------- |
|
63 |
// Stub Code definitions |
|
64 |
||
65 |
// Platform dependent parameters for array copy stubs |
|
66 |
||
67 |
// Note: we have noticed a huge change in behavior on a microbenchmark |
|
68 |
// from platform to platform depending on the configuration. |
|
69 |
||
70 |
// Instead of adding a series of command line options (which |
|
71 |
// unfortunately have to be done in the shared file and cannot appear |
|
72 |
// only in the ARM port), the tested result are hard-coded here in a set |
|
73 |
// of options, selected by specifying 'ArmCopyPlatform' |
|
74 |
||
75 |
// Currently, this 'platform' is hardcoded to a value that is a good |
|
76 |
// enough trade-off. However, one can easily modify this file to test |
|
77 |
// the hard-coded configurations or create new ones. If the gain is |
|
78 |
// significant, we could decide to either add command line options or |
|
79 |
// add code to automatically choose a configuration. |
|
80 |
||
81 |
// see comments below for the various configurations created |
|
82 |
#define DEFAULT_ARRAYCOPY_CONFIG 0 |
|
83 |
#define TEGRA2_ARRAYCOPY_CONFIG 1 |
|
84 |
#define IMX515_ARRAYCOPY_CONFIG 2 |
|
85 |
||
86 |
// Hard coded choices (XXX: could be changed to a command line option) |
|
87 |
#define ArmCopyPlatform DEFAULT_ARRAYCOPY_CONFIG |
|
88 |
||
89 |
#define ArmCopyCacheLineSize 32 // not worth optimizing to 64 according to measured gains |
|
90 |
||
91 |
// configuration for each kind of loop |
|
92 |
typedef struct { |
|
93 |
int pld_distance; // prefetch distance (0 => no prefetch, <0: prefetch_before); |
|
94 |
bool split_ldm; // if true, split each STM in STMs with fewer registers |
|
95 |
bool split_stm; // if true, split each LTM in LTMs with fewer registers |
|
96 |
} arraycopy_loop_config; |
|
97 |
||
98 |
// configuration for all loops |
|
99 |
typedef struct { |
|
100 |
// const char *description; |
|
101 |
arraycopy_loop_config forward_aligned; |
|
102 |
arraycopy_loop_config backward_aligned; |
|
103 |
arraycopy_loop_config forward_shifted; |
|
104 |
arraycopy_loop_config backward_shifted; |
|
105 |
} arraycopy_platform_config; |
|
106 |
||
107 |
// configured platforms |
|
108 |
static arraycopy_platform_config arraycopy_configurations[] = { |
|
109 |
// configuration parameters for arraycopy loops |
|
110 |
||
111 |
// Configurations were chosen based on manual analysis of benchmark |
|
112 |
// results, minimizing overhead with respect to best results on the |
|
113 |
// different test cases. |
|
114 |
||
115 |
// Prefetch before is always favored since it avoids dirtying the |
|
116 |
// cache uselessly for small copies. Code for prefetch after has |
|
117 |
// been kept in case the difference is significant for some |
|
118 |
// platforms but we might consider dropping it. |
|
119 |
||
120 |
// distance, ldm, stm |
|
121 |
{ |
|
122 |
// default: tradeoff tegra2/imx515/nv-tegra2, |
|
123 |
// Notes on benchmarking: |
|
124 |
// - not far from optimal configuration on nv-tegra2 |
|
125 |
// - within 5% of optimal configuration except for backward aligned on IMX |
|
126 |
// - up to 40% from optimal configuration for backward shifted and backward align for tegra2 |
|
127 |
// but still on par with the operating system copy |
|
128 |
{-256, true, true }, // forward aligned |
|
129 |
{-256, true, true }, // backward aligned |
|
130 |
{-256, false, false }, // forward shifted |
|
131 |
{-256, true, true } // backward shifted |
|
132 |
}, |
|
133 |
{ |
|
134 |
// configuration tuned on tegra2-4. |
|
135 |
// Warning: should not be used on nv-tegra2 ! |
|
136 |
// Notes: |
|
137 |
// - prefetch after gives 40% gain on backward copies on tegra2-4, |
|
138 |
// resulting in better number than the operating system |
|
139 |
// copy. However, this can lead to a 300% loss on nv-tegra and has |
|
140 |
// more impact on the cache (fetches futher than what is |
|
141 |
// copied). Use this configuration with care, in case it improves |
|
142 |
// reference benchmarks. |
|
143 |
{-256, true, true }, // forward aligned |
|
144 |
{96, false, false }, // backward aligned |
|
145 |
{-256, false, false }, // forward shifted |
|
146 |
{96, false, false } // backward shifted |
|
147 |
}, |
|
148 |
{ |
|
149 |
// configuration tuned on imx515 |
|
150 |
// Notes: |
|
151 |
// - smaller prefetch distance is sufficient to get good result and might be more stable |
|
152 |
// - refined backward aligned options within 5% of optimal configuration except for |
|
153 |
// tests were the arrays fit in the cache |
|
154 |
{-160, false, false }, // forward aligned |
|
155 |
{-160, false, false }, // backward aligned |
|
156 |
{-160, false, false }, // forward shifted |
|
157 |
{-160, true, true } // backward shifted |
|
158 |
} |
|
159 |
}; |
|
160 |
||
161 |
class StubGenerator: public StubCodeGenerator { |
|
162 |
||
163 |
#ifdef PRODUCT |
|
164 |
#define inc_counter_np(a,b,c) ((void)0) |
|
165 |
#else |
|
166 |
#define inc_counter_np(counter, t1, t2) \ |
|
167 |
BLOCK_COMMENT("inc_counter " #counter); \ |
|
168 |
__ inc_counter(&counter, t1, t2); |
|
169 |
#endif |
|
170 |
||
171 |
private: |
|
172 |
||
173 |
address generate_call_stub(address& return_address) { |
|
174 |
StubCodeMark mark(this, "StubRoutines", "call_stub"); |
|
175 |
address start = __ pc(); |
|
176 |
||
177 |
||
178 |
assert(frame::entry_frame_call_wrapper_offset == 0, "adjust this code"); |
|
179 |
||
180 |
__ mov(Rtemp, SP); |
|
181 |
__ push(RegisterSet(FP) | RegisterSet(LR)); |
|
182 |
#ifndef __SOFTFP__ |
|
183 |
__ fstmdbd(SP, FloatRegisterSet(D8, 8), writeback); |
|
184 |
#endif |
|
185 |
__ stmdb(SP, RegisterSet(R0, R2) | RegisterSet(R4, R6) | RegisterSet(R8, R10) | altFP_7_11, writeback); |
|
186 |
__ mov(Rmethod, R3); |
|
187 |
__ ldmia(Rtemp, RegisterSet(R1, R3) | Rthread); // stacked arguments |
|
188 |
||
189 |
// XXX: TODO |
|
190 |
// Would be better with respect to native tools if the following |
|
191 |
// setting of FP was changed to conform to the native ABI, with FP |
|
192 |
// pointing to the saved FP slot (and the corresponding modifications |
|
193 |
// for entry_frame_call_wrapper_offset and frame::real_fp). |
|
194 |
__ mov(FP, SP); |
|
195 |
||
196 |
{ |
|
197 |
Label no_parameters, pass_parameters; |
|
198 |
__ cmp(R3, 0); |
|
199 |
__ b(no_parameters, eq); |
|
200 |
||
201 |
__ bind(pass_parameters); |
|
202 |
__ ldr(Rtemp, Address(R2, wordSize, post_indexed)); // Rtemp OK, unused and scratchable |
|
203 |
__ subs(R3, R3, 1); |
|
204 |
__ push(Rtemp); |
|
205 |
__ b(pass_parameters, ne); |
|
206 |
__ bind(no_parameters); |
|
207 |
} |
|
208 |
||
209 |
__ mov(Rsender_sp, SP); |
|
210 |
__ blx(R1); |
|
211 |
return_address = __ pc(); |
|
212 |
||
213 |
__ add(SP, FP, wordSize); // Skip link to JavaCallWrapper |
|
214 |
__ pop(RegisterSet(R2, R3)); |
|
215 |
#ifndef __ABI_HARD__ |
|
216 |
__ cmp(R3, T_LONG); |
|
217 |
__ cmp(R3, T_DOUBLE, ne); |
|
218 |
__ str(R0, Address(R2)); |
|
219 |
__ str(R1, Address(R2, wordSize), eq); |
|
220 |
#else |
|
221 |
Label cont, l_float, l_double; |
|
222 |
||
223 |
__ cmp(R3, T_DOUBLE); |
|
224 |
__ b(l_double, eq); |
|
225 |
||
226 |
__ cmp(R3, T_FLOAT); |
|
227 |
__ b(l_float, eq); |
|
228 |
||
229 |
__ cmp(R3, T_LONG); |
|
230 |
__ str(R0, Address(R2)); |
|
231 |
__ str(R1, Address(R2, wordSize), eq); |
|
232 |
__ b(cont); |
|
233 |
||
234 |
||
235 |
__ bind(l_double); |
|
236 |
__ fstd(D0, Address(R2)); |
|
237 |
__ b(cont); |
|
238 |
||
239 |
__ bind(l_float); |
|
240 |
__ fsts(S0, Address(R2)); |
|
241 |
||
242 |
__ bind(cont); |
|
243 |
#endif |
|
244 |
||
245 |
__ pop(RegisterSet(R4, R6) | RegisterSet(R8, R10) | altFP_7_11); |
|
246 |
#ifndef __SOFTFP__ |
|
247 |
__ fldmiad(SP, FloatRegisterSet(D8, 8), writeback); |
|
248 |
#endif |
|
249 |
__ pop(RegisterSet(FP) | RegisterSet(PC)); |
|
250 |
||
251 |
return start; |
|
252 |
} |
|
253 |
||
254 |
||
255 |
// (in) Rexception_obj: exception oop |
|
256 |
address generate_catch_exception() { |
|
257 |
StubCodeMark mark(this, "StubRoutines", "catch_exception"); |
|
258 |
address start = __ pc(); |
|
259 |
||
260 |
__ str(Rexception_obj, Address(Rthread, Thread::pending_exception_offset())); |
|
261 |
__ b(StubRoutines::_call_stub_return_address); |
|
262 |
||
263 |
return start; |
|
264 |
} |
|
265 |
||
266 |
||
267 |
// (in) Rexception_pc: return address |
|
268 |
address generate_forward_exception() { |
|
269 |
StubCodeMark mark(this, "StubRoutines", "forward exception"); |
|
270 |
address start = __ pc(); |
|
271 |
||
272 |
__ mov(c_rarg0, Rthread); |
|
273 |
__ mov(c_rarg1, Rexception_pc); |
|
274 |
__ call_VM_leaf(CAST_FROM_FN_PTR(address, |
|
275 |
SharedRuntime::exception_handler_for_return_address), |
|
276 |
c_rarg0, c_rarg1); |
|
277 |
__ ldr(Rexception_obj, Address(Rthread, Thread::pending_exception_offset())); |
|
278 |
const Register Rzero = __ zero_register(Rtemp); // Rtemp OK (cleared by above call) |
|
279 |
__ str(Rzero, Address(Rthread, Thread::pending_exception_offset())); |
|
280 |
||
281 |
#ifdef ASSERT |
|
282 |
// make sure exception is set |
|
283 |
{ Label L; |
|
284 |
__ cbnz(Rexception_obj, L); |
|
285 |
__ stop("StubRoutines::forward exception: no pending exception (2)"); |
|
286 |
__ bind(L); |
|
287 |
} |
|
288 |
#endif |
|
289 |
||
290 |
// Verify that there is really a valid exception in RAX. |
|
291 |
__ verify_oop(Rexception_obj); |
|
292 |
||
293 |
__ jump(R0); // handler is returned in R0 by runtime function |
|
294 |
return start; |
|
295 |
} |
|
296 |
||
297 |
||
298 |
||
299 |
// Integer division shared routine |
|
300 |
// Input: |
|
301 |
// R0 - dividend |
|
302 |
// R2 - divisor |
|
303 |
// Output: |
|
304 |
// R0 - remainder |
|
305 |
// R1 - quotient |
|
306 |
// Destroys: |
|
307 |
// R2 |
|
308 |
// LR |
|
309 |
address generate_idiv_irem() { |
|
310 |
Label positive_arguments, negative_or_zero, call_slow_path; |
|
311 |
Register dividend = R0; |
|
312 |
Register divisor = R2; |
|
313 |
Register remainder = R0; |
|
314 |
Register quotient = R1; |
|
315 |
Register tmp = LR; |
|
316 |
assert(dividend == remainder, "must be"); |
|
317 |
||
318 |
address start = __ pc(); |
|
319 |
||
320 |
// Check for special cases: divisor <= 0 or dividend < 0 |
|
321 |
__ cmp(divisor, 0); |
|
322 |
__ orrs(quotient, dividend, divisor, ne); |
|
323 |
__ b(negative_or_zero, le); |
|
324 |
||
325 |
__ bind(positive_arguments); |
|
326 |
// Save return address on stack to free one extra register |
|
327 |
__ push(LR); |
|
328 |
// Approximate the mamximum order of the quotient |
|
329 |
__ clz(tmp, dividend); |
|
330 |
__ clz(quotient, divisor); |
|
331 |
__ subs(tmp, quotient, tmp); |
|
332 |
__ mov(quotient, 0); |
|
333 |
// Jump to the appropriate place in the unrolled loop below |
|
334 |
__ ldr(PC, Address(PC, tmp, lsl, 2), pl); |
|
335 |
// If divisor is greater than dividend, return immediately |
|
336 |
__ pop(PC); |
|
337 |
||
338 |
// Offset table |
|
339 |
Label offset_table[32]; |
|
340 |
int i; |
|
341 |
for (i = 0; i <= 31; i++) { |
|
342 |
__ emit_address(offset_table[i]); |
|
343 |
} |
|
344 |
||
345 |
// Unrolled loop of 32 division steps |
|
346 |
for (i = 31; i >= 0; i--) { |
|
347 |
__ bind(offset_table[i]); |
|
348 |
__ cmp(remainder, AsmOperand(divisor, lsl, i)); |
|
349 |
__ sub(remainder, remainder, AsmOperand(divisor, lsl, i), hs); |
|
350 |
__ add(quotient, quotient, 1 << i, hs); |
|
351 |
} |
|
352 |
__ pop(PC); |
|
353 |
||
354 |
__ bind(negative_or_zero); |
|
355 |
// Find the combination of argument signs and jump to corresponding handler |
|
356 |
__ andr(quotient, dividend, 0x80000000, ne); |
|
357 |
__ orr(quotient, quotient, AsmOperand(divisor, lsr, 31), ne); |
|
358 |
__ add(PC, PC, AsmOperand(quotient, ror, 26), ne); |
|
359 |
__ str(LR, Address(Rthread, JavaThread::saved_exception_pc_offset())); |
|
360 |
||
361 |
// The leaf runtime function can destroy R0-R3 and R12 registers which are still alive |
|
362 |
RegisterSet saved_registers = RegisterSet(R3) | RegisterSet(R12); |
|
363 |
#if R9_IS_SCRATCHED |
|
364 |
// Safer to save R9 here since callers may have been written |
|
365 |
// assuming R9 survives. This is suboptimal but may not be worth |
|
366 |
// revisiting for this slow case. |
|
367 |
||
368 |
// save also R10 for alignment |
|
369 |
saved_registers = saved_registers | RegisterSet(R9, R10); |
|
370 |
#endif |
|
371 |
{ |
|
372 |
// divisor == 0 |
|
373 |
FixedSizeCodeBlock zero_divisor(_masm, 8, true); |
|
374 |
__ push(saved_registers); |
|
375 |
__ mov(R0, Rthread); |
|
376 |
__ mov(R1, LR); |
|
377 |
__ mov(R2, SharedRuntime::IMPLICIT_DIVIDE_BY_ZERO); |
|
378 |
__ b(call_slow_path); |
|
379 |
} |
|
380 |
||
381 |
{ |
|
382 |
// divisor > 0 && dividend < 0 |
|
383 |
FixedSizeCodeBlock positive_divisor_negative_dividend(_masm, 8, true); |
|
384 |
__ push(LR); |
|
385 |
__ rsb(dividend, dividend, 0); |
|
386 |
__ bl(positive_arguments); |
|
387 |
__ rsb(remainder, remainder, 0); |
|
388 |
__ rsb(quotient, quotient, 0); |
|
389 |
__ pop(PC); |
|
390 |
} |
|
391 |
||
392 |
{ |
|
393 |
// divisor < 0 && dividend > 0 |
|
394 |
FixedSizeCodeBlock negative_divisor_positive_dividend(_masm, 8, true); |
|
395 |
__ push(LR); |
|
396 |
__ rsb(divisor, divisor, 0); |
|
397 |
__ bl(positive_arguments); |
|
398 |
__ rsb(quotient, quotient, 0); |
|
399 |
__ pop(PC); |
|
400 |
} |
|
401 |
||
402 |
{ |
|
403 |
// divisor < 0 && dividend < 0 |
|
404 |
FixedSizeCodeBlock negative_divisor_negative_dividend(_masm, 8, true); |
|
405 |
__ push(LR); |
|
406 |
__ rsb(dividend, dividend, 0); |
|
407 |
__ rsb(divisor, divisor, 0); |
|
408 |
__ bl(positive_arguments); |
|
409 |
__ rsb(remainder, remainder, 0); |
|
410 |
__ pop(PC); |
|
411 |
} |
|
412 |
||
413 |
__ bind(call_slow_path); |
|
414 |
__ call(CAST_FROM_FN_PTR(address, SharedRuntime::continuation_for_implicit_exception)); |
|
415 |
__ pop(saved_registers); |
|
416 |
__ bx(R0); |
|
417 |
||
418 |
return start; |
|
419 |
} |
|
420 |
||
421 |
||
422 |
// As per atomic.hpp the Atomic read-modify-write operations must be logically implemented as: |
|
423 |
// <fence>; <op>; <membar StoreLoad|StoreStore> |
|
424 |
// But for load-linked/store-conditional based systems a fence here simply means |
|
425 |
// no load/store can be reordered with respect to the initial load-linked, so we have: |
|
426 |
// <membar storeload|loadload> ; load-linked; <op>; store-conditional; <membar storeload|storestore> |
|
427 |
// There are no memory actions in <op> so nothing further is needed. |
|
428 |
// |
|
429 |
// So we define the following for convenience: |
|
430 |
#define MEMBAR_ATOMIC_OP_PRE \ |
|
431 |
MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad|MacroAssembler::LoadLoad) |
|
432 |
#define MEMBAR_ATOMIC_OP_POST \ |
|
433 |
MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad|MacroAssembler::StoreStore) |
|
434 |
||
435 |
// Note: JDK 9 only supports ARMv7+ so we always have ldrexd available even though the |
|
436 |
// code below allows for it to be otherwise. The else clause indicates an ARMv5 system |
|
437 |
// for which we do not support MP and so membars are not necessary. This ARMv5 code will |
|
438 |
// be removed in the future. |
|
439 |
||
59249
29b0d0b61615
8234737: Harmonize parameter order in Atomic - add
stefank
parents:
55490
diff
changeset
|
440 |
// Implementation of atomic_add(jint add_value, volatile jint* dest) |
29b0d0b61615
8234737: Harmonize parameter order in Atomic - add
stefank
parents:
55490
diff
changeset
|
441 |
// used by Atomic::add(volatile jint* dest, jint add_value) |
42664 | 442 |
// |
443 |
// Arguments : |
|
444 |
// |
|
445 |
// add_value: R0 |
|
446 |
// dest: R1 |
|
447 |
// |
|
448 |
// Results: |
|
449 |
// |
|
450 |
// R0: the new stored in dest |
|
451 |
// |
|
452 |
// Overwrites: |
|
453 |
// |
|
454 |
// R1, R2, R3 |
|
455 |
// |
|
456 |
address generate_atomic_add() { |
|
457 |
address start; |
|
458 |
||
459 |
StubCodeMark mark(this, "StubRoutines", "atomic_add"); |
|
460 |
Label retry; |
|
461 |
start = __ pc(); |
|
462 |
Register addval = R0; |
|
463 |
Register dest = R1; |
|
464 |
Register prev = R2; |
|
465 |
Register ok = R2; |
|
466 |
Register newval = R3; |
|
467 |
||
468 |
if (VM_Version::supports_ldrex()) { |
|
469 |
__ membar(MEMBAR_ATOMIC_OP_PRE, prev); |
|
470 |
__ bind(retry); |
|
471 |
__ ldrex(newval, Address(dest)); |
|
472 |
__ add(newval, addval, newval); |
|
473 |
__ strex(ok, newval, Address(dest)); |
|
474 |
__ cmp(ok, 0); |
|
475 |
__ b(retry, ne); |
|
476 |
__ mov (R0, newval); |
|
477 |
__ membar(MEMBAR_ATOMIC_OP_POST, prev); |
|
478 |
} else { |
|
479 |
__ bind(retry); |
|
480 |
__ ldr (prev, Address(dest)); |
|
481 |
__ add(newval, addval, prev); |
|
482 |
__ atomic_cas_bool(prev, newval, dest, 0, noreg/*ignored*/); |
|
483 |
__ b(retry, ne); |
|
484 |
__ mov (R0, newval); |
|
485 |
} |
|
486 |
__ bx(LR); |
|
487 |
||
488 |
return start; |
|
489 |
} |
|
490 |
||
491 |
// Support for jint Atomic::xchg(jint exchange_value, volatile jint *dest) |
|
492 |
// |
|
493 |
// Arguments : |
|
494 |
// |
|
495 |
// exchange_value: R0 |
|
496 |
// dest: R1 |
|
497 |
// |
|
498 |
// Results: |
|
499 |
// |
|
500 |
// R0: the value previously stored in dest |
|
501 |
// |
|
502 |
// Overwrites: |
|
503 |
// |
|
504 |
// R1, R2, R3 |
|
505 |
// |
|
506 |
address generate_atomic_xchg() { |
|
507 |
address start; |
|
508 |
||
509 |
StubCodeMark mark(this, "StubRoutines", "atomic_xchg"); |
|
510 |
start = __ pc(); |
|
511 |
Register newval = R0; |
|
512 |
Register dest = R1; |
|
513 |
Register prev = R2; |
|
514 |
||
515 |
Label retry; |
|
516 |
||
517 |
if (VM_Version::supports_ldrex()) { |
|
518 |
Register ok=R3; |
|
519 |
__ membar(MEMBAR_ATOMIC_OP_PRE, prev); |
|
520 |
__ bind(retry); |
|
521 |
__ ldrex(prev, Address(dest)); |
|
522 |
__ strex(ok, newval, Address(dest)); |
|
523 |
__ cmp(ok, 0); |
|
524 |
__ b(retry, ne); |
|
525 |
__ mov (R0, prev); |
|
526 |
__ membar(MEMBAR_ATOMIC_OP_POST, prev); |
|
527 |
} else { |
|
528 |
__ bind(retry); |
|
529 |
__ ldr (prev, Address(dest)); |
|
530 |
__ atomic_cas_bool(prev, newval, dest, 0, noreg/*ignored*/); |
|
531 |
__ b(retry, ne); |
|
532 |
__ mov (R0, prev); |
|
533 |
} |
|
534 |
__ bx(LR); |
|
535 |
||
536 |
return start; |
|
537 |
} |
|
538 |
||
539 |
// Support for jint Atomic::cmpxchg(jint exchange_value, volatile jint *dest, jint compare_value) |
|
540 |
// |
|
541 |
// Arguments : |
|
542 |
// |
|
543 |
// compare_value: R0 |
|
544 |
// exchange_value: R1 |
|
545 |
// dest: R2 |
|
546 |
// |
|
547 |
// Results: |
|
548 |
// |
|
549 |
// R0: the value previously stored in dest |
|
550 |
// |
|
551 |
// Overwrites: |
|
552 |
// |
|
553 |
// R0, R1, R2, R3, Rtemp |
|
554 |
// |
|
555 |
address generate_atomic_cmpxchg() { |
|
556 |
address start; |
|
557 |
||
558 |
StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg"); |
|
559 |
start = __ pc(); |
|
560 |
Register cmp = R0; |
|
561 |
Register newval = R1; |
|
562 |
Register dest = R2; |
|
563 |
Register temp1 = R3; |
|
564 |
Register temp2 = Rtemp; // Rtemp free (native ABI) |
|
565 |
||
566 |
__ membar(MEMBAR_ATOMIC_OP_PRE, temp1); |
|
567 |
||
568 |
// atomic_cas returns previous value in R0 |
|
569 |
__ atomic_cas(temp1, temp2, cmp, newval, dest, 0); |
|
570 |
||
571 |
__ membar(MEMBAR_ATOMIC_OP_POST, temp1); |
|
572 |
||
573 |
__ bx(LR); |
|
574 |
||
575 |
return start; |
|
576 |
} |
|
577 |
||
578 |
// Support for jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong *dest, jlong compare_value) |
|
579 |
// reordered before by a wrapper to (jlong compare_value, jlong exchange_value, volatile jlong *dest) |
|
580 |
// |
|
581 |
// Arguments : |
|
582 |
// |
|
583 |
// compare_value: R1 (High), R0 (Low) |
|
584 |
// exchange_value: R3 (High), R2 (Low) |
|
585 |
// dest: SP+0 |
|
586 |
// |
|
587 |
// Results: |
|
588 |
// |
|
589 |
// R0:R1: the value previously stored in dest |
|
590 |
// |
|
591 |
// Overwrites: |
|
592 |
// |
|
593 |
address generate_atomic_cmpxchg_long() { |
|
594 |
address start; |
|
595 |
||
596 |
StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg_long"); |
|
597 |
start = __ pc(); |
|
598 |
Register cmp_lo = R0; |
|
599 |
Register cmp_hi = R1; |
|
600 |
Register newval_lo = R2; |
|
601 |
Register newval_hi = R3; |
|
602 |
Register addr = Rtemp; /* After load from stack */ |
|
603 |
Register temp_lo = R4; |
|
604 |
Register temp_hi = R5; |
|
605 |
Register temp_result = R8; |
|
606 |
assert_different_registers(cmp_lo, newval_lo, temp_lo, addr, temp_result, R7); |
|
607 |
assert_different_registers(cmp_hi, newval_hi, temp_hi, addr, temp_result, R7); |
|
608 |
||
609 |
__ membar(MEMBAR_ATOMIC_OP_PRE, Rtemp); // Rtemp free (native ABI) |
|
610 |
||
611 |
// Stack is unaligned, maintain double word alignment by pushing |
|
612 |
// odd number of regs. |
|
613 |
__ push(RegisterSet(temp_result) | RegisterSet(temp_lo, temp_hi)); |
|
614 |
__ ldr(addr, Address(SP, 12)); |
|
615 |
||
616 |
// atomic_cas64 returns previous value in temp_lo, temp_hi |
|
617 |
__ atomic_cas64(temp_lo, temp_hi, temp_result, cmp_lo, cmp_hi, |
|
618 |
newval_lo, newval_hi, addr, 0); |
|
619 |
__ mov(R0, temp_lo); |
|
620 |
__ mov(R1, temp_hi); |
|
621 |
||
622 |
__ pop(RegisterSet(temp_result) | RegisterSet(temp_lo, temp_hi)); |
|
623 |
||
624 |
__ membar(MEMBAR_ATOMIC_OP_POST, Rtemp); // Rtemp free (native ABI) |
|
625 |
__ bx(LR); |
|
626 |
||
627 |
return start; |
|
628 |
} |
|
629 |
||
630 |
address generate_atomic_load_long() { |
|
631 |
address start; |
|
632 |
||
633 |
StubCodeMark mark(this, "StubRoutines", "atomic_load_long"); |
|
634 |
start = __ pc(); |
|
635 |
Register result_lo = R0; |
|
636 |
Register result_hi = R1; |
|
637 |
Register src = R0; |
|
638 |
||
639 |
if (!os::is_MP()) { |
|
640 |
__ ldmia(src, RegisterSet(result_lo, result_hi)); |
|
641 |
__ bx(LR); |
|
642 |
} else if (VM_Version::supports_ldrexd()) { |
|
643 |
__ ldrexd(result_lo, Address(src)); |
|
644 |
__ clrex(); // FIXME: safe to remove? |
|
645 |
__ bx(LR); |
|
646 |
} else { |
|
647 |
__ stop("Atomic load(jlong) unsupported on this platform"); |
|
648 |
__ bx(LR); |
|
649 |
} |
|
650 |
||
651 |
return start; |
|
652 |
} |
|
653 |
||
654 |
address generate_atomic_store_long() { |
|
655 |
address start; |
|
656 |
||
657 |
StubCodeMark mark(this, "StubRoutines", "atomic_store_long"); |
|
658 |
start = __ pc(); |
|
659 |
Register newval_lo = R0; |
|
660 |
Register newval_hi = R1; |
|
661 |
Register dest = R2; |
|
662 |
Register scratch_lo = R2; |
|
663 |
Register scratch_hi = R3; /* After load from stack */ |
|
664 |
Register result = R3; |
|
665 |
||
666 |
if (!os::is_MP()) { |
|
667 |
__ stmia(dest, RegisterSet(newval_lo, newval_hi)); |
|
668 |
__ bx(LR); |
|
669 |
} else if (VM_Version::supports_ldrexd()) { |
|
670 |
__ mov(Rtemp, dest); // get dest to Rtemp |
|
671 |
Label retry; |
|
672 |
__ bind(retry); |
|
673 |
__ ldrexd(scratch_lo, Address(Rtemp)); |
|
674 |
__ strexd(result, R0, Address(Rtemp)); |
|
675 |
__ rsbs(result, result, 1); |
|
676 |
__ b(retry, eq); |
|
677 |
__ bx(LR); |
|
678 |
} else { |
|
679 |
__ stop("Atomic store(jlong) unsupported on this platform"); |
|
680 |
__ bx(LR); |
|
681 |
} |
|
682 |
||
683 |
return start; |
|
684 |
} |
|
685 |
||
686 |
||
687 |
||
688 |
#ifdef COMPILER2 |
|
689 |
// Support for uint StubRoutine::Arm::partial_subtype_check( Klass sub, Klass super ); |
|
690 |
// Arguments : |
|
691 |
// |
|
692 |
// ret : R0, returned |
|
693 |
// icc/xcc: set as R0 (depending on wordSize) |
|
694 |
// sub : R1, argument, not changed |
|
695 |
// super: R2, argument, not changed |
|
696 |
// raddr: LR, blown by call |
|
697 |
address generate_partial_subtype_check() { |
|
698 |
__ align(CodeEntryAlignment); |
|
699 |
StubCodeMark mark(this, "StubRoutines", "partial_subtype_check"); |
|
700 |
address start = __ pc(); |
|
701 |
||
702 |
// based on SPARC check_klass_subtype_[fast|slow]_path (without CompressedOops) |
|
703 |
||
704 |
// R0 used as tmp_reg (in addition to return reg) |
|
705 |
Register sub_klass = R1; |
|
706 |
Register super_klass = R2; |
|
707 |
Register tmp_reg2 = R3; |
|
708 |
Register tmp_reg3 = R4; |
|
709 |
#define saved_set tmp_reg2, tmp_reg3 |
|
710 |
||
711 |
Label L_loop, L_fail; |
|
712 |
||
713 |
int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); |
|
714 |
||
715 |
// fast check should be redundant |
|
716 |
||
717 |
// slow check |
|
718 |
{ |
|
719 |
__ raw_push(saved_set); |
|
720 |
||
721 |
// a couple of useful fields in sub_klass: |
|
722 |
int ss_offset = in_bytes(Klass::secondary_supers_offset()); |
|
723 |
||
724 |
// Do a linear scan of the secondary super-klass chain. |
|
725 |
// This code is rarely used, so simplicity is a virtue here. |
|
726 |
||
727 |
inc_counter_np(SharedRuntime::_partial_subtype_ctr, tmp_reg2, tmp_reg3); |
|
728 |
||
729 |
Register scan_temp = tmp_reg2; |
|
730 |
Register count_temp = tmp_reg3; |
|
731 |
||
732 |
// We will consult the secondary-super array. |
|
733 |
__ ldr(scan_temp, Address(sub_klass, ss_offset)); |
|
734 |
||
735 |
Register search_key = super_klass; |
|
736 |
||
737 |
// Load the array length. |
|
738 |
__ ldr_s32(count_temp, Address(scan_temp, Array<Klass*>::length_offset_in_bytes())); |
|
739 |
__ add(scan_temp, scan_temp, Array<Klass*>::base_offset_in_bytes()); |
|
740 |
||
741 |
__ add(count_temp, count_temp, 1); |
|
742 |
||
743 |
// Top of search loop |
|
744 |
__ bind(L_loop); |
|
745 |
// Notes: |
|
746 |
// scan_temp starts at the array elements |
|
747 |
// count_temp is 1+size |
|
748 |
__ subs(count_temp, count_temp, 1); |
|
749 |
__ b(L_fail, eq); // not found in the array |
|
750 |
||
751 |
// Load next super to check |
|
752 |
// In the array of super classes elements are pointer sized. |
|
753 |
int element_size = wordSize; |
|
754 |
__ ldr(R0, Address(scan_temp, element_size, post_indexed)); |
|
755 |
||
756 |
// Look for Rsuper_klass on Rsub_klass's secondary super-class-overflow list |
|
757 |
__ subs(R0, R0, search_key); // set R0 to 0 on success (and flags to eq) |
|
758 |
||
759 |
// A miss means we are NOT a subtype and need to keep looping |
|
760 |
__ b(L_loop, ne); |
|
761 |
||
762 |
// Falling out the bottom means we found a hit; we ARE a subtype |
|
763 |
||
764 |
// Success. Cache the super we found and proceed in triumph. |
|
765 |
__ str(super_klass, Address(sub_klass, sc_offset)); |
|
766 |
||
767 |
// Return success |
|
768 |
// R0 is already 0 and flags are already set to eq |
|
769 |
__ raw_pop(saved_set); |
|
770 |
__ ret(); |
|
771 |
||
772 |
// Return failure |
|
773 |
__ bind(L_fail); |
|
774 |
__ movs(R0, 1); // sets the flags |
|
775 |
__ raw_pop(saved_set); |
|
776 |
__ ret(); |
|
777 |
} |
|
778 |
return start; |
|
779 |
} |
|
780 |
#undef saved_set |
|
781 |
#endif // COMPILER2 |
|
782 |
||
783 |
||
784 |
//---------------------------------------------------------------------------------------------------- |
|
785 |
// Non-destructive plausibility checks for oops |
|
786 |
||
787 |
address generate_verify_oop() { |
|
788 |
StubCodeMark mark(this, "StubRoutines", "verify_oop"); |
|
789 |
address start = __ pc(); |
|
790 |
||
791 |
// Incoming arguments: |
|
792 |
// |
|
793 |
// R0: error message (char* ) |
|
794 |
// R1: address of register save area |
|
795 |
// R2: oop to verify |
|
796 |
// |
|
797 |
// All registers are saved before calling this stub. However, condition flags should be saved here. |
|
798 |
||
799 |
const Register oop = R2; |
|
800 |
const Register klass = R3; |
|
801 |
const Register tmp1 = R6; |
|
802 |
const Register tmp2 = R8; |
|
803 |
||
804 |
const Register flags = Rtmp_save0; // R4/R19 |
|
805 |
const Register ret_addr = Rtmp_save1; // R5/R20 |
|
806 |
assert_different_registers(oop, klass, tmp1, tmp2, flags, ret_addr, R7); |
|
807 |
||
808 |
Label exit, error; |
|
809 |
InlinedAddress verify_oop_count((address) StubRoutines::verify_oop_count_addr()); |
|
810 |
||
811 |
__ mrs(Assembler::CPSR, flags); |
|
812 |
||
813 |
__ ldr_literal(tmp1, verify_oop_count); |
|
814 |
__ ldr_s32(tmp2, Address(tmp1)); |
|
815 |
__ add(tmp2, tmp2, 1); |
|
816 |
__ str_32(tmp2, Address(tmp1)); |
|
817 |
||
818 |
// make sure object is 'reasonable' |
|
819 |
__ cbz(oop, exit); // if obj is NULL it is ok |
|
820 |
||
821 |
// Check if the oop is in the right area of memory |
|
822 |
// Note: oop_mask and oop_bits must be updated if the code is saved/reused |
|
823 |
const address oop_mask = (address) Universe::verify_oop_mask(); |
|
824 |
const address oop_bits = (address) Universe::verify_oop_bits(); |
|
53686 | 825 |
__ mov_address(tmp1, oop_mask); |
42664 | 826 |
__ andr(tmp2, oop, tmp1); |
53686 | 827 |
__ mov_address(tmp1, oop_bits); |
42664 | 828 |
__ cmp(tmp2, tmp1); |
829 |
__ b(error, ne); |
|
830 |
||
831 |
// make sure klass is 'reasonable' |
|
832 |
__ load_klass(klass, oop); // get klass |
|
833 |
__ cbz(klass, error); // if klass is NULL it is broken |
|
834 |
||
835 |
// return if everything seems ok |
|
836 |
__ bind(exit); |
|
837 |
||
838 |
__ msr(Assembler::CPSR_f, flags); |
|
839 |
||
840 |
__ ret(); |
|
841 |
||
842 |
// handle errors |
|
843 |
__ bind(error); |
|
844 |
||
845 |
__ mov(ret_addr, LR); // save return address |
|
846 |
||
847 |
// R0: error message |
|
848 |
// R1: register save area |
|
849 |
__ call(CAST_FROM_FN_PTR(address, MacroAssembler::debug)); |
|
850 |
||
851 |
__ mov(LR, ret_addr); |
|
852 |
__ b(exit); |
|
853 |
||
854 |
__ bind_literal(verify_oop_count); |
|
855 |
||
856 |
return start; |
|
857 |
} |
|
858 |
||
859 |
//---------------------------------------------------------------------------------------------------- |
|
860 |
// Array copy stubs |
|
861 |
||
862 |
// |
|
863 |
// Generate overlap test for array copy stubs |
|
864 |
// |
|
865 |
// Input: |
|
866 |
// R0 - array1 |
|
867 |
// R1 - array2 |
|
868 |
// R2 - element count, 32-bit int |
|
869 |
// |
|
870 |
// input registers are preserved |
|
871 |
// |
|
872 |
void array_overlap_test(address no_overlap_target, int log2_elem_size, Register tmp1, Register tmp2) { |
|
873 |
assert(no_overlap_target != NULL, "must be generated"); |
|
874 |
array_overlap_test(no_overlap_target, NULL, log2_elem_size, tmp1, tmp2); |
|
875 |
} |
|
876 |
void array_overlap_test(Label& L_no_overlap, int log2_elem_size, Register tmp1, Register tmp2) { |
|
877 |
array_overlap_test(NULL, &L_no_overlap, log2_elem_size, tmp1, tmp2); |
|
878 |
} |
|
879 |
void array_overlap_test(address no_overlap_target, Label* NOLp, int log2_elem_size, Register tmp1, Register tmp2) { |
|
880 |
const Register from = R0; |
|
881 |
const Register to = R1; |
|
882 |
const Register count = R2; |
|
883 |
const Register to_from = tmp1; // to - from |
|
884 |
const Register byte_count = (log2_elem_size == 0) ? count : tmp2; // count << log2_elem_size |
|
885 |
assert_different_registers(from, to, count, tmp1, tmp2); |
|
886 |
||
887 |
// no_overlap version works if 'to' lower (unsigned) than 'from' |
|
888 |
// and or 'to' more than (count*size) from 'from' |
|
889 |
||
890 |
BLOCK_COMMENT("Array Overlap Test:"); |
|
891 |
__ subs(to_from, to, from); |
|
892 |
if (log2_elem_size != 0) { |
|
893 |
__ mov(byte_count, AsmOperand(count, lsl, log2_elem_size)); |
|
894 |
} |
|
895 |
if (NOLp == NULL) |
|
896 |
__ b(no_overlap_target,lo); |
|
897 |
else |
|
898 |
__ b((*NOLp), lo); |
|
899 |
__ cmp(to_from, byte_count); |
|
900 |
if (NOLp == NULL) |
|
901 |
__ b(no_overlap_target, ge); |
|
902 |
else |
|
903 |
__ b((*NOLp), ge); |
|
904 |
} |
|
905 |
||
52351 | 906 |
|
42664 | 907 |
// probably we should choose between "prefetch-store before or after store", not "before or after load". |
908 |
void prefetch(Register from, Register to, int offset, int to_delta = 0) { |
|
909 |
__ prefetch_read(Address(from, offset)); |
|
910 |
} |
|
911 |
||
912 |
// Generate the inner loop for forward aligned array copy |
|
913 |
// |
|
914 |
// Arguments |
|
915 |
// from: src address, 64 bits aligned |
|
916 |
// to: dst address, wordSize aligned |
|
917 |
// count: number of elements (32-bit int) |
|
918 |
// bytes_per_count: number of bytes for each unit of 'count' |
|
919 |
// |
|
920 |
// Return the minimum initial value for count |
|
921 |
// |
|
922 |
// Notes: |
|
52351 | 923 |
// - 'from' aligned on 64-bit (recommended for 32-bit ARM in case this speeds up LDMIA) |
42664 | 924 |
// - 'to' aligned on wordSize |
925 |
// - 'count' must be greater or equal than the returned value |
|
926 |
// |
|
927 |
// Increases 'from' and 'to' by count*bytes_per_count. |
|
928 |
// |
|
929 |
// Scratches 'count', R3. |
|
52351 | 930 |
// R4-R10 are preserved (saved/restored). |
42664 | 931 |
// |
55490
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
932 |
int generate_forward_aligned_copy_loop(Register from, Register to, Register count, int bytes_per_count, bool unsafe_copy = false) { |
42664 | 933 |
assert (from == R0 && to == R1 && count == R2, "adjust the implementation below"); |
934 |
||
935 |
const int bytes_per_loop = 8*wordSize; // 8 registers are read and written on every loop iteration |
|
936 |
arraycopy_loop_config *config=&arraycopy_configurations[ArmCopyPlatform].forward_aligned; |
|
937 |
int pld_offset = config->pld_distance; |
|
938 |
const int count_per_loop = bytes_per_loop / bytes_per_count; |
|
939 |
||
940 |
bool split_read= config->split_ldm; |
|
941 |
bool split_write= config->split_stm; |
|
942 |
||
943 |
// XXX optim: use VLDM/VSTM when available (Neon) with PLD |
|
944 |
// NEONCopyPLD |
|
945 |
// PLD [r1, #0xC0] |
|
946 |
// VLDM r1!,{d0-d7} |
|
947 |
// VSTM r0!,{d0-d7} |
|
948 |
// SUBS r2,r2,#0x40 |
|
949 |
// BGE NEONCopyPLD |
|
950 |
||
951 |
__ push(RegisterSet(R4,R10)); |
|
952 |
||
953 |
const bool prefetch_before = pld_offset < 0; |
|
954 |
const bool prefetch_after = pld_offset > 0; |
|
955 |
||
956 |
Label L_skip_pld; |
|
957 |
||
958 |
{ |
|
55490
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
959 |
// UnsafeCopyMemory page error: continue after ucm |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
960 |
UnsafeCopyMemoryMark ucmm(this, unsafe_copy, true); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
961 |
// predecrease to exit when there is less than count_per_loop |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
962 |
__ sub_32(count, count, count_per_loop); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
963 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
964 |
if (pld_offset != 0) { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
965 |
pld_offset = (pld_offset < 0) ? -pld_offset : pld_offset; |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
966 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
967 |
prefetch(from, to, 0); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
968 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
969 |
if (prefetch_before) { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
970 |
// If prefetch is done ahead, final PLDs that overflow the |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
971 |
// copied area can be easily avoided. 'count' is predecreased |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
972 |
// by the prefetch distance to optimize the inner loop and the |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
973 |
// outer loop skips the PLD. |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
974 |
__ subs_32(count, count, (bytes_per_loop+pld_offset)/bytes_per_count); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
975 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
976 |
// skip prefetch for small copies |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
977 |
__ b(L_skip_pld, lt); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
978 |
} |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
979 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
980 |
int offset = ArmCopyCacheLineSize; |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
981 |
while (offset <= pld_offset) { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
982 |
prefetch(from, to, offset); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
983 |
offset += ArmCopyCacheLineSize; |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
984 |
}; |
42664 | 985 |
} |
986 |
||
55490
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
987 |
{ |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
988 |
// 32-bit ARM note: we have tried implementing loop unrolling to skip one |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
989 |
// PLD with 64 bytes cache line but the gain was not significant. |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
990 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
991 |
Label L_copy_loop; |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
992 |
__ align(OptoLoopAlignment); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
993 |
__ BIND(L_copy_loop); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
994 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
995 |
if (prefetch_before) { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
996 |
prefetch(from, to, bytes_per_loop + pld_offset); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
997 |
__ BIND(L_skip_pld); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
998 |
} |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
999 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1000 |
if (split_read) { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1001 |
// Split the register set in two sets so that there is less |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1002 |
// latency between LDM and STM (R3-R6 available while R7-R10 |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1003 |
// still loading) and less register locking issue when iterating |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1004 |
// on the first LDM. |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1005 |
__ ldmia(from, RegisterSet(R3, R6), writeback); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1006 |
__ ldmia(from, RegisterSet(R7, R10), writeback); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1007 |
} else { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1008 |
__ ldmia(from, RegisterSet(R3, R10), writeback); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1009 |
} |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1010 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1011 |
__ subs_32(count, count, count_per_loop); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1012 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1013 |
if (prefetch_after) { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1014 |
prefetch(from, to, pld_offset, bytes_per_loop); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1015 |
} |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1016 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1017 |
if (split_write) { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1018 |
__ stmia(to, RegisterSet(R3, R6), writeback); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1019 |
__ stmia(to, RegisterSet(R7, R10), writeback); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1020 |
} else { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1021 |
__ stmia(to, RegisterSet(R3, R10), writeback); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1022 |
} |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1023 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1024 |
__ b(L_copy_loop, ge); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1025 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1026 |
if (prefetch_before) { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1027 |
// the inner loop may end earlier, allowing to skip PLD for the last iterations |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1028 |
__ cmn_32(count, (bytes_per_loop + pld_offset)/bytes_per_count); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1029 |
__ b(L_skip_pld, ge); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1030 |
} |
42664 | 1031 |
} |
55490
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1032 |
BLOCK_COMMENT("Remaining bytes:"); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1033 |
// still 0..bytes_per_loop-1 aligned bytes to copy, count already decreased by (at least) bytes_per_loop bytes |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1034 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1035 |
// __ add(count, count, ...); // addition useless for the bit tests |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1036 |
assert (pld_offset % bytes_per_loop == 0, "decreasing count by pld_offset before loop must not change tested bits"); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1037 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1038 |
__ tst(count, 16 / bytes_per_count); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1039 |
__ ldmia(from, RegisterSet(R3, R6), writeback, ne); // copy 16 bytes |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1040 |
__ stmia(to, RegisterSet(R3, R6), writeback, ne); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1041 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1042 |
__ tst(count, 8 / bytes_per_count); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1043 |
__ ldmia(from, RegisterSet(R3, R4), writeback, ne); // copy 8 bytes |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1044 |
__ stmia(to, RegisterSet(R3, R4), writeback, ne); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1045 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1046 |
if (bytes_per_count <= 4) { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1047 |
__ tst(count, 4 / bytes_per_count); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1048 |
__ ldr(R3, Address(from, 4, post_indexed), ne); // copy 4 bytes |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1049 |
__ str(R3, Address(to, 4, post_indexed), ne); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1050 |
} |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1051 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1052 |
if (bytes_per_count <= 2) { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1053 |
__ tst(count, 2 / bytes_per_count); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1054 |
__ ldrh(R3, Address(from, 2, post_indexed), ne); // copy 2 bytes |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1055 |
__ strh(R3, Address(to, 2, post_indexed), ne); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1056 |
} |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1057 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1058 |
if (bytes_per_count == 1) { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1059 |
__ tst(count, 1); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1060 |
__ ldrb(R3, Address(from, 1, post_indexed), ne); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1061 |
__ strb(R3, Address(to, 1, post_indexed), ne); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1062 |
} |
42664 | 1063 |
} |
1064 |
||
1065 |
__ pop(RegisterSet(R4,R10)); |
|
1066 |
||
1067 |
return count_per_loop; |
|
1068 |
} |
|
1069 |
||
1070 |
||
1071 |
// Generate the inner loop for backward aligned array copy |
|
1072 |
// |
|
1073 |
// Arguments |
|
1074 |
// end_from: src end address, 64 bits aligned |
|
1075 |
// end_to: dst end address, wordSize aligned |
|
1076 |
// count: number of elements (32-bit int) |
|
1077 |
// bytes_per_count: number of bytes for each unit of 'count' |
|
1078 |
// |
|
1079 |
// Return the minimum initial value for count |
|
1080 |
// |
|
1081 |
// Notes: |
|
52351 | 1082 |
// - 'end_from' aligned on 64-bit (recommended for 32-bit ARM in case this speeds up LDMIA) |
42664 | 1083 |
// - 'end_to' aligned on wordSize |
1084 |
// - 'count' must be greater or equal than the returned value |
|
1085 |
// |
|
1086 |
// Decreases 'end_from' and 'end_to' by count*bytes_per_count. |
|
1087 |
// |
|
1088 |
// Scratches 'count', R3. |
|
52351 | 1089 |
// ARM R4-R10 are preserved (saved/restored). |
42664 | 1090 |
// |
55490
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1091 |
int generate_backward_aligned_copy_loop(Register end_from, Register end_to, Register count, int bytes_per_count, bool unsafe_copy = false) { |
42664 | 1092 |
assert (end_from == R0 && end_to == R1 && count == R2, "adjust the implementation below"); |
1093 |
||
1094 |
const int bytes_per_loop = 8*wordSize; // 8 registers are read and written on every loop iteration |
|
1095 |
const int count_per_loop = bytes_per_loop / bytes_per_count; |
|
1096 |
||
1097 |
arraycopy_loop_config *config=&arraycopy_configurations[ArmCopyPlatform].backward_aligned; |
|
1098 |
int pld_offset = config->pld_distance; |
|
1099 |
||
1100 |
bool split_read= config->split_ldm; |
|
1101 |
bool split_write= config->split_stm; |
|
1102 |
||
1103 |
// See the forward copy variant for additional comments. |
|
1104 |
||
1105 |
__ push(RegisterSet(R4,R10)); |
|
1106 |
||
55490
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1107 |
{ |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1108 |
// UnsafeCopyMemory page error: continue after ucm |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1109 |
UnsafeCopyMemoryMark ucmm(this, unsafe_copy, true); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1110 |
__ sub_32(count, count, count_per_loop); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1111 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1112 |
const bool prefetch_before = pld_offset < 0; |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1113 |
const bool prefetch_after = pld_offset > 0; |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1114 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1115 |
Label L_skip_pld; |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1116 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1117 |
if (pld_offset != 0) { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1118 |
pld_offset = (pld_offset < 0) ? -pld_offset : pld_offset; |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1119 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1120 |
prefetch(end_from, end_to, -wordSize); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1121 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1122 |
if (prefetch_before) { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1123 |
__ subs_32(count, count, (bytes_per_loop + pld_offset) / bytes_per_count); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1124 |
__ b(L_skip_pld, lt); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1125 |
} |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1126 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1127 |
int offset = ArmCopyCacheLineSize; |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1128 |
while (offset <= pld_offset) { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1129 |
prefetch(end_from, end_to, -(wordSize + offset)); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1130 |
offset += ArmCopyCacheLineSize; |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1131 |
}; |
42664 | 1132 |
} |
1133 |
||
55490
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1134 |
{ |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1135 |
// 32-bit ARM note: we have tried implementing loop unrolling to skip one |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1136 |
// PLD with 64 bytes cache line but the gain was not significant. |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1137 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1138 |
Label L_copy_loop; |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1139 |
__ align(OptoLoopAlignment); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1140 |
__ BIND(L_copy_loop); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1141 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1142 |
if (prefetch_before) { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1143 |
prefetch(end_from, end_to, -(wordSize + bytes_per_loop + pld_offset)); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1144 |
__ BIND(L_skip_pld); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1145 |
} |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1146 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1147 |
if (split_read) { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1148 |
__ ldmdb(end_from, RegisterSet(R7, R10), writeback); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1149 |
__ ldmdb(end_from, RegisterSet(R3, R6), writeback); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1150 |
} else { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1151 |
__ ldmdb(end_from, RegisterSet(R3, R10), writeback); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1152 |
} |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1153 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1154 |
__ subs_32(count, count, count_per_loop); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1155 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1156 |
if (prefetch_after) { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1157 |
prefetch(end_from, end_to, -(wordSize + pld_offset), -bytes_per_loop); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1158 |
} |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1159 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1160 |
if (split_write) { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1161 |
__ stmdb(end_to, RegisterSet(R7, R10), writeback); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1162 |
__ stmdb(end_to, RegisterSet(R3, R6), writeback); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1163 |
} else { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1164 |
__ stmdb(end_to, RegisterSet(R3, R10), writeback); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1165 |
} |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1166 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1167 |
__ b(L_copy_loop, ge); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1168 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1169 |
if (prefetch_before) { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1170 |
__ cmn_32(count, (bytes_per_loop + pld_offset)/bytes_per_count); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1171 |
__ b(L_skip_pld, ge); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1172 |
} |
42664 | 1173 |
} |
55490
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1174 |
BLOCK_COMMENT("Remaining bytes:"); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1175 |
// still 0..bytes_per_loop-1 aligned bytes to copy, count already decreased by (at least) bytes_per_loop bytes |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1176 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1177 |
// __ add(count, count, ...); // addition useless for the bit tests |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1178 |
assert (pld_offset % bytes_per_loop == 0, "decreasing count by pld_offset before loop must not change tested bits"); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1179 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1180 |
__ tst(count, 16 / bytes_per_count); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1181 |
__ ldmdb(end_from, RegisterSet(R3, R6), writeback, ne); // copy 16 bytes |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1182 |
__ stmdb(end_to, RegisterSet(R3, R6), writeback, ne); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1183 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1184 |
__ tst(count, 8 / bytes_per_count); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1185 |
__ ldmdb(end_from, RegisterSet(R3, R4), writeback, ne); // copy 8 bytes |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1186 |
__ stmdb(end_to, RegisterSet(R3, R4), writeback, ne); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1187 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1188 |
if (bytes_per_count <= 4) { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1189 |
__ tst(count, 4 / bytes_per_count); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1190 |
__ ldr(R3, Address(end_from, -4, pre_indexed), ne); // copy 4 bytes |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1191 |
__ str(R3, Address(end_to, -4, pre_indexed), ne); |
42664 | 1192 |
} |
1193 |
||
55490
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1194 |
if (bytes_per_count <= 2) { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1195 |
__ tst(count, 2 / bytes_per_count); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1196 |
__ ldrh(R3, Address(end_from, -2, pre_indexed), ne); // copy 2 bytes |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1197 |
__ strh(R3, Address(end_to, -2, pre_indexed), ne); |
42664 | 1198 |
} |
1199 |
||
55490
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1200 |
if (bytes_per_count == 1) { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1201 |
__ tst(count, 1); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1202 |
__ ldrb(R3, Address(end_from, -1, pre_indexed), ne); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1203 |
__ strb(R3, Address(end_to, -1, pre_indexed), ne); |
42664 | 1204 |
} |
1205 |
} |
|
1206 |
__ pop(RegisterSet(R4,R10)); |
|
1207 |
||
1208 |
return count_per_loop; |
|
1209 |
} |
|
1210 |
||
1211 |
||
1212 |
// Generate the inner loop for shifted forward array copy (unaligned copy). |
|
52351 | 1213 |
// It can be used when bytes_per_count < wordSize, i.e. byte/short copy |
42664 | 1214 |
// |
1215 |
// Arguments |
|
1216 |
// from: start src address, 64 bits aligned |
|
1217 |
// to: start dst address, (now) wordSize aligned |
|
1218 |
// count: number of elements (32-bit int) |
|
1219 |
// bytes_per_count: number of bytes for each unit of 'count' |
|
1220 |
// lsr_shift: shift applied to 'old' value to skipped already written bytes |
|
1221 |
// lsl_shift: shift applied to 'new' value to set the high bytes of the next write |
|
1222 |
// |
|
1223 |
// Return the minimum initial value for count |
|
1224 |
// |
|
1225 |
// Notes: |
|
52351 | 1226 |
// - 'from' aligned on 64-bit (recommended for 32-bit ARM in case this speeds up LDMIA) |
42664 | 1227 |
// - 'to' aligned on wordSize |
1228 |
// - 'count' must be greater or equal than the returned value |
|
1229 |
// - 'lsr_shift' + 'lsl_shift' = BitsPerWord |
|
52351 | 1230 |
// - 'bytes_per_count' is 1 or 2 |
42664 | 1231 |
// |
1232 |
// Increases 'to' by count*bytes_per_count. |
|
1233 |
// |
|
1234 |
// Scratches 'from' and 'count', R3-R10, R12 |
|
1235 |
// |
|
1236 |
// On entry: |
|
1237 |
// - R12 is preloaded with the first 'BitsPerWord' bits read just before 'from' |
|
1238 |
// - (R12 >> lsr_shift) is the part not yet written (just before 'to') |
|
1239 |
// --> (*to) = (R12 >> lsr_shift) | (*from) << lsl_shift); ... |
|
1240 |
// |
|
1241 |
// This implementation may read more bytes than required. |
|
1242 |
// Actually, it always reads exactly all data from the copied region with upper bound aligned up by wordSize, |
|
1243 |
// so excessive read do not cross a word bound and is thus harmless. |
|
1244 |
// |
|
1245 |
int generate_forward_shifted_copy_loop(Register from, Register to, Register count, int bytes_per_count, int lsr_shift, int lsl_shift) { |
|
1246 |
assert (from == R0 && to == R1 && count == R2, "adjust the implementation below"); |
|
1247 |
||
1248 |
const int bytes_per_loop = 8*wordSize; // 8 registers are read and written on every loop iter |
|
1249 |
const int count_per_loop = bytes_per_loop / bytes_per_count; |
|
1250 |
||
1251 |
arraycopy_loop_config *config=&arraycopy_configurations[ArmCopyPlatform].forward_shifted; |
|
1252 |
int pld_offset = config->pld_distance; |
|
1253 |
||
1254 |
bool split_read= config->split_ldm; |
|
1255 |
bool split_write= config->split_stm; |
|
1256 |
||
1257 |
const bool prefetch_before = pld_offset < 0; |
|
1258 |
const bool prefetch_after = pld_offset > 0; |
|
1259 |
Label L_skip_pld, L_last_read, L_done; |
|
1260 |
if (pld_offset != 0) { |
|
1261 |
||
1262 |
pld_offset = (pld_offset < 0) ? -pld_offset : pld_offset; |
|
1263 |
||
1264 |
prefetch(from, to, 0); |
|
1265 |
||
1266 |
if (prefetch_before) { |
|
1267 |
__ cmp_32(count, count_per_loop); |
|
1268 |
__ b(L_last_read, lt); |
|
1269 |
// skip prefetch for small copies |
|
1270 |
// warning: count is predecreased by the prefetch distance to optimize the inner loop |
|
1271 |
__ subs_32(count, count, ((bytes_per_loop + pld_offset) / bytes_per_count) + count_per_loop); |
|
1272 |
__ b(L_skip_pld, lt); |
|
1273 |
} |
|
1274 |
||
1275 |
int offset = ArmCopyCacheLineSize; |
|
1276 |
while (offset <= pld_offset) { |
|
1277 |
prefetch(from, to, offset); |
|
1278 |
offset += ArmCopyCacheLineSize; |
|
1279 |
}; |
|
1280 |
} |
|
1281 |
||
1282 |
Label L_shifted_loop; |
|
1283 |
||
1284 |
__ align(OptoLoopAlignment); |
|
1285 |
__ BIND(L_shifted_loop); |
|
1286 |
||
1287 |
if (prefetch_before) { |
|
1288 |
// do it early if there might be register locking issues |
|
1289 |
prefetch(from, to, bytes_per_loop + pld_offset); |
|
1290 |
__ BIND(L_skip_pld); |
|
1291 |
} else { |
|
1292 |
__ cmp_32(count, count_per_loop); |
|
1293 |
__ b(L_last_read, lt); |
|
1294 |
} |
|
1295 |
||
1296 |
// read 32 bytes |
|
1297 |
if (split_read) { |
|
1298 |
// if write is not split, use less registers in first set to reduce locking |
|
1299 |
RegisterSet set1 = split_write ? RegisterSet(R4, R7) : RegisterSet(R4, R5); |
|
1300 |
RegisterSet set2 = (split_write ? RegisterSet(R8, R10) : RegisterSet(R6, R10)) | R12; |
|
1301 |
__ ldmia(from, set1, writeback); |
|
1302 |
__ mov(R3, AsmOperand(R12, lsr, lsr_shift)); // part of R12 not yet written |
|
1303 |
__ ldmia(from, set2, writeback); |
|
1304 |
__ subs(count, count, count_per_loop); // XXX: should it be before the 2nd LDM ? (latency vs locking) |
|
1305 |
} else { |
|
1306 |
__ mov(R3, AsmOperand(R12, lsr, lsr_shift)); // part of R12 not yet written |
|
1307 |
__ ldmia(from, RegisterSet(R4, R10) | R12, writeback); // Note: small latency on R4 |
|
1308 |
__ subs(count, count, count_per_loop); |
|
1309 |
} |
|
1310 |
||
1311 |
if (prefetch_after) { |
|
1312 |
// do it after the 1st ldm/ldp anyway (no locking issues with early STM/STP) |
|
1313 |
prefetch(from, to, pld_offset, bytes_per_loop); |
|
1314 |
} |
|
1315 |
||
1316 |
// prepare (shift) the values in R3..R10 |
|
1317 |
__ orr(R3, R3, AsmOperand(R4, lsl, lsl_shift)); // merged below low bytes of next val |
|
1318 |
__ logical_shift_right(R4, R4, lsr_shift); // unused part of next val |
|
1319 |
__ orr(R4, R4, AsmOperand(R5, lsl, lsl_shift)); // ... |
|
1320 |
__ logical_shift_right(R5, R5, lsr_shift); |
|
1321 |
__ orr(R5, R5, AsmOperand(R6, lsl, lsl_shift)); |
|
1322 |
__ logical_shift_right(R6, R6, lsr_shift); |
|
1323 |
__ orr(R6, R6, AsmOperand(R7, lsl, lsl_shift)); |
|
1324 |
if (split_write) { |
|
1325 |
// write the first half as soon as possible to reduce stm locking |
|
1326 |
__ stmia(to, RegisterSet(R3, R6), writeback, prefetch_before ? gt : ge); |
|
1327 |
} |
|
1328 |
__ logical_shift_right(R7, R7, lsr_shift); |
|
1329 |
__ orr(R7, R7, AsmOperand(R8, lsl, lsl_shift)); |
|
1330 |
__ logical_shift_right(R8, R8, lsr_shift); |
|
1331 |
__ orr(R8, R8, AsmOperand(R9, lsl, lsl_shift)); |
|
1332 |
__ logical_shift_right(R9, R9, lsr_shift); |
|
1333 |
__ orr(R9, R9, AsmOperand(R10, lsl, lsl_shift)); |
|
1334 |
__ logical_shift_right(R10, R10, lsr_shift); |
|
1335 |
__ orr(R10, R10, AsmOperand(R12, lsl, lsl_shift)); |
|
1336 |
||
1337 |
if (split_write) { |
|
1338 |
__ stmia(to, RegisterSet(R7, R10), writeback, prefetch_before ? gt : ge); |
|
1339 |
} else { |
|
1340 |
__ stmia(to, RegisterSet(R3, R10), writeback, prefetch_before ? gt : ge); |
|
1341 |
} |
|
1342 |
__ b(L_shifted_loop, gt); // no need to loop if 0 (when count need not be precise modulo bytes_per_loop) |
|
1343 |
||
1344 |
if (prefetch_before) { |
|
1345 |
// the first loop may end earlier, allowing to skip pld at the end |
|
1346 |
__ cmn_32(count, (bytes_per_loop + pld_offset)/bytes_per_count); |
|
1347 |
__ stmia(to, RegisterSet(R3, R10), writeback); // stmia was skipped |
|
1348 |
__ b(L_skip_pld, ge); |
|
1349 |
__ adds_32(count, count, ((bytes_per_loop + pld_offset) / bytes_per_count) + count_per_loop); |
|
1350 |
} |
|
1351 |
||
1352 |
__ BIND(L_last_read); |
|
1353 |
__ b(L_done, eq); |
|
1354 |
||
1355 |
switch (bytes_per_count) { |
|
1356 |
case 2: |
|
1357 |
__ mov(R3, AsmOperand(R12, lsr, lsr_shift)); |
|
1358 |
__ tst(count, 8); |
|
1359 |
__ ldmia(from, RegisterSet(R4, R7), writeback, ne); |
|
1360 |
__ orr(R3, R3, AsmOperand(R4, lsl, lsl_shift), ne); // merged below low bytes of next val |
|
1361 |
__ mov(R4, AsmOperand(R4, lsr, lsr_shift), ne); // unused part of next val |
|
1362 |
__ orr(R4, R4, AsmOperand(R5, lsl, lsl_shift), ne); // ... |
|
1363 |
__ mov(R5, AsmOperand(R5, lsr, lsr_shift), ne); |
|
1364 |
__ orr(R5, R5, AsmOperand(R6, lsl, lsl_shift), ne); |
|
1365 |
__ mov(R6, AsmOperand(R6, lsr, lsr_shift), ne); |
|
1366 |
__ orr(R6, R6, AsmOperand(R7, lsl, lsl_shift), ne); |
|
1367 |
__ stmia(to, RegisterSet(R3, R6), writeback, ne); |
|
1368 |
__ mov(R3, AsmOperand(R7, lsr, lsr_shift), ne); |
|
1369 |
||
1370 |
__ tst(count, 4); |
|
1371 |
__ ldmia(from, RegisterSet(R4, R5), writeback, ne); |
|
1372 |
__ orr(R3, R3, AsmOperand(R4, lsl, lsl_shift), ne); // merged below low bytes of next val |
|
1373 |
__ mov(R4, AsmOperand(R4, lsr, lsr_shift), ne); // unused part of next val |
|
1374 |
__ orr(R4, R4, AsmOperand(R5, lsl, lsl_shift), ne); // ... |
|
1375 |
__ stmia(to, RegisterSet(R3, R4), writeback, ne); |
|
1376 |
__ mov(R3, AsmOperand(R5, lsr, lsr_shift), ne); |
|
1377 |
||
1378 |
__ tst(count, 2); |
|
1379 |
__ ldr(R4, Address(from, 4, post_indexed), ne); |
|
1380 |
__ orr(R3, R3, AsmOperand(R4, lsl, lsl_shift), ne); |
|
1381 |
__ str(R3, Address(to, 4, post_indexed), ne); |
|
1382 |
__ mov(R3, AsmOperand(R4, lsr, lsr_shift), ne); |
|
1383 |
||
1384 |
__ tst(count, 1); |
|
1385 |
__ strh(R3, Address(to, 2, post_indexed), ne); // one last short |
|
1386 |
break; |
|
1387 |
||
1388 |
case 1: |
|
1389 |
__ mov(R3, AsmOperand(R12, lsr, lsr_shift)); |
|
1390 |
__ tst(count, 16); |
|
1391 |
__ ldmia(from, RegisterSet(R4, R7), writeback, ne); |
|
1392 |
__ orr(R3, R3, AsmOperand(R4, lsl, lsl_shift), ne); // merged below low bytes of next val |
|
1393 |
__ mov(R4, AsmOperand(R4, lsr, lsr_shift), ne); // unused part of next val |
|
1394 |
__ orr(R4, R4, AsmOperand(R5, lsl, lsl_shift), ne); // ... |
|
1395 |
__ mov(R5, AsmOperand(R5, lsr, lsr_shift), ne); |
|
1396 |
__ orr(R5, R5, AsmOperand(R6, lsl, lsl_shift), ne); |
|
1397 |
__ mov(R6, AsmOperand(R6, lsr, lsr_shift), ne); |
|
1398 |
__ orr(R6, R6, AsmOperand(R7, lsl, lsl_shift), ne); |
|
1399 |
__ stmia(to, RegisterSet(R3, R6), writeback, ne); |
|
1400 |
__ mov(R3, AsmOperand(R7, lsr, lsr_shift), ne); |
|
1401 |
||
1402 |
__ tst(count, 8); |
|
1403 |
__ ldmia(from, RegisterSet(R4, R5), writeback, ne); |
|
1404 |
__ orr(R3, R3, AsmOperand(R4, lsl, lsl_shift), ne); // merged below low bytes of next val |
|
1405 |
__ mov(R4, AsmOperand(R4, lsr, lsr_shift), ne); // unused part of next val |
|
1406 |
__ orr(R4, R4, AsmOperand(R5, lsl, lsl_shift), ne); // ... |
|
1407 |
__ stmia(to, RegisterSet(R3, R4), writeback, ne); |
|
1408 |
__ mov(R3, AsmOperand(R5, lsr, lsr_shift), ne); |
|
1409 |
||
1410 |
__ tst(count, 4); |
|
1411 |
__ ldr(R4, Address(from, 4, post_indexed), ne); |
|
1412 |
__ orr(R3, R3, AsmOperand(R4, lsl, lsl_shift), ne); |
|
1413 |
__ str(R3, Address(to, 4, post_indexed), ne); |
|
1414 |
__ mov(R3, AsmOperand(R4, lsr, lsr_shift), ne); |
|
1415 |
||
1416 |
__ andr(count, count, 3); |
|
1417 |
__ cmp(count, 2); |
|
1418 |
||
1419 |
// Note: R3 might contain enough bytes ready to write (3 needed at most), |
|
1420 |
// thus load on lsl_shift==24 is not needed (in fact forces reading |
|
1421 |
// beyond source buffer end boundary) |
|
1422 |
if (lsl_shift == 8) { |
|
1423 |
__ ldr(R4, Address(from, 4, post_indexed), ge); |
|
1424 |
__ orr(R3, R3, AsmOperand(R4, lsl, lsl_shift), ge); |
|
1425 |
} else if (lsl_shift == 16) { |
|
1426 |
__ ldr(R4, Address(from, 4, post_indexed), gt); |
|
1427 |
__ orr(R3, R3, AsmOperand(R4, lsl, lsl_shift), gt); |
|
1428 |
} |
|
1429 |
||
1430 |
__ strh(R3, Address(to, 2, post_indexed), ge); // two last bytes |
|
1431 |
__ mov(R3, AsmOperand(R3, lsr, 16), gt); |
|
1432 |
||
1433 |
__ tst(count, 1); |
|
1434 |
__ strb(R3, Address(to, 1, post_indexed), ne); // one last byte |
|
1435 |
break; |
|
1436 |
} |
|
1437 |
||
1438 |
__ BIND(L_done); |
|
1439 |
return 0; // no minimum |
|
1440 |
} |
|
1441 |
||
1442 |
// Generate the inner loop for shifted backward array copy (unaligned copy). |
|
52351 | 1443 |
// It can be used when bytes_per_count < wordSize, i.e. byte/short copy |
42664 | 1444 |
// |
1445 |
// Arguments |
|
1446 |
// end_from: end src address, 64 bits aligned |
|
1447 |
// end_to: end dst address, (now) wordSize aligned |
|
1448 |
// count: number of elements (32-bit int) |
|
1449 |
// bytes_per_count: number of bytes for each unit of 'count' |
|
1450 |
// lsl_shift: shift applied to 'old' value to skipped already written bytes |
|
1451 |
// lsr_shift: shift applied to 'new' value to set the low bytes of the next write |
|
1452 |
// |
|
1453 |
// Return the minimum initial value for count |
|
1454 |
// |
|
1455 |
// Notes: |
|
52351 | 1456 |
// - 'end_from' aligned on 64-bit (recommended for 32-bit ARM in case this speeds up LDMIA) |
42664 | 1457 |
// - 'end_to' aligned on wordSize |
1458 |
// - 'count' must be greater or equal than the returned value |
|
1459 |
// - 'lsr_shift' + 'lsl_shift' = 'BitsPerWord' |
|
52351 | 1460 |
// - 'bytes_per_count' is 1 or 2 on 32-bit ARM |
42664 | 1461 |
// |
1462 |
// Decreases 'end_to' by count*bytes_per_count. |
|
1463 |
// |
|
1464 |
// Scratches 'end_from', 'count', R3-R10, R12 |
|
1465 |
// |
|
1466 |
// On entry: |
|
1467 |
// - R3 is preloaded with the first 'BitsPerWord' bits read just after 'from' |
|
1468 |
// - (R3 << lsl_shift) is the part not yet written |
|
1469 |
// --> (*--to) = (R3 << lsl_shift) | (*--from) >> lsr_shift); ... |
|
1470 |
// |
|
1471 |
// This implementation may read more bytes than required. |
|
1472 |
// Actually, it always reads exactly all data from the copied region with beginning aligned down by wordSize, |
|
1473 |
// so excessive read do not cross a word bound and is thus harmless. |
|
1474 |
// |
|
1475 |
int generate_backward_shifted_copy_loop(Register end_from, Register end_to, Register count, int bytes_per_count, int lsr_shift, int lsl_shift) { |
|
1476 |
assert (end_from == R0 && end_to == R1 && count == R2, "adjust the implementation below"); |
|
1477 |
||
1478 |
const int bytes_per_loop = 8*wordSize; // 8 registers are read and written on every loop iter |
|
1479 |
const int count_per_loop = bytes_per_loop / bytes_per_count; |
|
1480 |
||
1481 |
arraycopy_loop_config *config=&arraycopy_configurations[ArmCopyPlatform].backward_shifted; |
|
1482 |
int pld_offset = config->pld_distance; |
|
1483 |
||
1484 |
bool split_read= config->split_ldm; |
|
1485 |
bool split_write= config->split_stm; |
|
1486 |
||
1487 |
||
1488 |
const bool prefetch_before = pld_offset < 0; |
|
1489 |
const bool prefetch_after = pld_offset > 0; |
|
1490 |
||
1491 |
Label L_skip_pld, L_done, L_last_read; |
|
1492 |
if (pld_offset != 0) { |
|
1493 |
||
1494 |
pld_offset = (pld_offset < 0) ? -pld_offset : pld_offset; |
|
1495 |
||
1496 |
prefetch(end_from, end_to, -wordSize); |
|
1497 |
||
1498 |
if (prefetch_before) { |
|
1499 |
__ cmp_32(count, count_per_loop); |
|
1500 |
__ b(L_last_read, lt); |
|
1501 |
||
1502 |
// skip prefetch for small copies |
|
1503 |
// warning: count is predecreased by the prefetch distance to optimize the inner loop |
|
1504 |
__ subs_32(count, count, ((bytes_per_loop + pld_offset)/bytes_per_count) + count_per_loop); |
|
1505 |
__ b(L_skip_pld, lt); |
|
1506 |
} |
|
1507 |
||
1508 |
int offset = ArmCopyCacheLineSize; |
|
1509 |
while (offset <= pld_offset) { |
|
1510 |
prefetch(end_from, end_to, -(wordSize + offset)); |
|
1511 |
offset += ArmCopyCacheLineSize; |
|
1512 |
}; |
|
1513 |
} |
|
1514 |
||
1515 |
Label L_shifted_loop; |
|
1516 |
__ align(OptoLoopAlignment); |
|
1517 |
__ BIND(L_shifted_loop); |
|
1518 |
||
1519 |
if (prefetch_before) { |
|
1520 |
// do the 1st ldm/ldp first anyway (no locking issues with early STM/STP) |
|
1521 |
prefetch(end_from, end_to, -(wordSize + bytes_per_loop + pld_offset)); |
|
1522 |
__ BIND(L_skip_pld); |
|
1523 |
} else { |
|
1524 |
__ cmp_32(count, count_per_loop); |
|
1525 |
__ b(L_last_read, lt); |
|
1526 |
} |
|
1527 |
||
1528 |
if (split_read) { |
|
1529 |
__ ldmdb(end_from, RegisterSet(R7, R10), writeback); |
|
1530 |
__ mov(R12, AsmOperand(R3, lsl, lsl_shift)); // part of R3 not yet written |
|
1531 |
__ ldmdb(end_from, RegisterSet(R3, R6), writeback); |
|
1532 |
} else { |
|
1533 |
__ mov(R12, AsmOperand(R3, lsl, lsl_shift)); // part of R3 not yet written |
|
1534 |
__ ldmdb(end_from, RegisterSet(R3, R10), writeback); |
|
1535 |
} |
|
1536 |
||
1537 |
__ subs_32(count, count, count_per_loop); |
|
1538 |
||
1539 |
if (prefetch_after) { // do prefetch during ldm/ldp latency |
|
1540 |
prefetch(end_from, end_to, -(wordSize + pld_offset), -bytes_per_loop); |
|
1541 |
} |
|
1542 |
||
1543 |
// prepare the values in R4..R10,R12 |
|
1544 |
__ orr(R12, R12, AsmOperand(R10, lsr, lsr_shift)); // merged above high bytes of prev val |
|
1545 |
__ logical_shift_left(R10, R10, lsl_shift); // unused part of prev val |
|
1546 |
__ orr(R10, R10, AsmOperand(R9, lsr, lsr_shift)); // ... |
|
1547 |
__ logical_shift_left(R9, R9, lsl_shift); |
|
1548 |
__ orr(R9, R9, AsmOperand(R8, lsr, lsr_shift)); |
|
1549 |
__ logical_shift_left(R8, R8, lsl_shift); |
|
1550 |
__ orr(R8, R8, AsmOperand(R7, lsr, lsr_shift)); |
|
1551 |
__ logical_shift_left(R7, R7, lsl_shift); |
|
1552 |
__ orr(R7, R7, AsmOperand(R6, lsr, lsr_shift)); |
|
1553 |
__ logical_shift_left(R6, R6, lsl_shift); |
|
1554 |
__ orr(R6, R6, AsmOperand(R5, lsr, lsr_shift)); |
|
1555 |
if (split_write) { |
|
1556 |
// store early to reduce locking issues |
|
1557 |
__ stmdb(end_to, RegisterSet(R6, R10) | R12, writeback, prefetch_before ? gt : ge); |
|
1558 |
} |
|
1559 |
__ logical_shift_left(R5, R5, lsl_shift); |
|
1560 |
__ orr(R5, R5, AsmOperand(R4, lsr, lsr_shift)); |
|
1561 |
__ logical_shift_left(R4, R4, lsl_shift); |
|
1562 |
__ orr(R4, R4, AsmOperand(R3, lsr, lsr_shift)); |
|
1563 |
||
1564 |
if (split_write) { |
|
1565 |
__ stmdb(end_to, RegisterSet(R4, R5), writeback, prefetch_before ? gt : ge); |
|
1566 |
} else { |
|
1567 |
__ stmdb(end_to, RegisterSet(R4, R10) | R12, writeback, prefetch_before ? gt : ge); |
|
1568 |
} |
|
1569 |
||
1570 |
__ b(L_shifted_loop, gt); // no need to loop if 0 (when count need not be precise modulo bytes_per_loop) |
|
1571 |
||
1572 |
if (prefetch_before) { |
|
1573 |
// the first loop may end earlier, allowing to skip pld at the end |
|
1574 |
__ cmn_32(count, ((bytes_per_loop + pld_offset)/bytes_per_count)); |
|
1575 |
__ stmdb(end_to, RegisterSet(R4, R10) | R12, writeback); // stmdb was skipped |
|
1576 |
__ b(L_skip_pld, ge); |
|
1577 |
__ adds_32(count, count, ((bytes_per_loop + pld_offset) / bytes_per_count) + count_per_loop); |
|
1578 |
} |
|
1579 |
||
1580 |
__ BIND(L_last_read); |
|
1581 |
__ b(L_done, eq); |
|
1582 |
||
1583 |
switch(bytes_per_count) { |
|
1584 |
case 2: |
|
1585 |
__ mov(R12, AsmOperand(R3, lsl, lsl_shift)); // part of R3 not yet written |
|
1586 |
__ tst(count, 8); |
|
1587 |
__ ldmdb(end_from, RegisterSet(R7,R10), writeback, ne); |
|
1588 |
__ orr(R12, R12, AsmOperand(R10, lsr, lsr_shift), ne); |
|
1589 |
__ mov(R10, AsmOperand(R10, lsl, lsl_shift),ne); // unused part of prev val |
|
1590 |
__ orr(R10, R10, AsmOperand(R9, lsr, lsr_shift),ne); // ... |
|
1591 |
__ mov(R9, AsmOperand(R9, lsl, lsl_shift),ne); |
|
1592 |
__ orr(R9, R9, AsmOperand(R8, lsr, lsr_shift),ne); |
|
1593 |
__ mov(R8, AsmOperand(R8, lsl, lsl_shift),ne); |
|
1594 |
__ orr(R8, R8, AsmOperand(R7, lsr, lsr_shift),ne); |
|
1595 |
__ stmdb(end_to, RegisterSet(R8,R10)|R12, writeback, ne); |
|
1596 |
__ mov(R12, AsmOperand(R7, lsl, lsl_shift), ne); |
|
1597 |
||
1598 |
__ tst(count, 4); |
|
1599 |
__ ldmdb(end_from, RegisterSet(R9, R10), writeback, ne); |
|
1600 |
__ orr(R12, R12, AsmOperand(R10, lsr, lsr_shift), ne); |
|
1601 |
__ mov(R10, AsmOperand(R10, lsl, lsl_shift),ne); // unused part of prev val |
|
1602 |
__ orr(R10, R10, AsmOperand(R9, lsr,lsr_shift),ne); // ... |
|
1603 |
__ stmdb(end_to, RegisterSet(R10)|R12, writeback, ne); |
|
1604 |
__ mov(R12, AsmOperand(R9, lsl, lsl_shift), ne); |
|
1605 |
||
1606 |
__ tst(count, 2); |
|
1607 |
__ ldr(R10, Address(end_from, -4, pre_indexed), ne); |
|
1608 |
__ orr(R12, R12, AsmOperand(R10, lsr, lsr_shift), ne); |
|
1609 |
__ str(R12, Address(end_to, -4, pre_indexed), ne); |
|
1610 |
__ mov(R12, AsmOperand(R10, lsl, lsl_shift), ne); |
|
1611 |
||
1612 |
__ tst(count, 1); |
|
1613 |
__ mov(R12, AsmOperand(R12, lsr, lsr_shift),ne); |
|
1614 |
__ strh(R12, Address(end_to, -2, pre_indexed), ne); // one last short |
|
1615 |
break; |
|
1616 |
||
1617 |
case 1: |
|
1618 |
__ mov(R12, AsmOperand(R3, lsl, lsl_shift)); // part of R3 not yet written |
|
1619 |
__ tst(count, 16); |
|
1620 |
__ ldmdb(end_from, RegisterSet(R7,R10), writeback, ne); |
|
1621 |
__ orr(R12, R12, AsmOperand(R10, lsr, lsr_shift), ne); |
|
1622 |
__ mov(R10, AsmOperand(R10, lsl, lsl_shift),ne); // unused part of prev val |
|
1623 |
__ orr(R10, R10, AsmOperand(R9, lsr, lsr_shift),ne); // ... |
|
1624 |
__ mov(R9, AsmOperand(R9, lsl, lsl_shift),ne); |
|
1625 |
__ orr(R9, R9, AsmOperand(R8, lsr, lsr_shift),ne); |
|
1626 |
__ mov(R8, AsmOperand(R8, lsl, lsl_shift),ne); |
|
1627 |
__ orr(R8, R8, AsmOperand(R7, lsr, lsr_shift),ne); |
|
1628 |
__ stmdb(end_to, RegisterSet(R8,R10)|R12, writeback, ne); |
|
1629 |
__ mov(R12, AsmOperand(R7, lsl, lsl_shift), ne); |
|
1630 |
||
1631 |
__ tst(count, 8); |
|
1632 |
__ ldmdb(end_from, RegisterSet(R9,R10), writeback, ne); |
|
1633 |
__ orr(R12, R12, AsmOperand(R10, lsr, lsr_shift), ne); |
|
1634 |
__ mov(R10, AsmOperand(R10, lsl, lsl_shift),ne); // unused part of prev val |
|
1635 |
__ orr(R10, R10, AsmOperand(R9, lsr, lsr_shift),ne); // ... |
|
1636 |
__ stmdb(end_to, RegisterSet(R10)|R12, writeback, ne); |
|
1637 |
__ mov(R12, AsmOperand(R9, lsl, lsl_shift), ne); |
|
1638 |
||
1639 |
__ tst(count, 4); |
|
1640 |
__ ldr(R10, Address(end_from, -4, pre_indexed), ne); |
|
1641 |
__ orr(R12, R12, AsmOperand(R10, lsr, lsr_shift), ne); |
|
1642 |
__ str(R12, Address(end_to, -4, pre_indexed), ne); |
|
1643 |
__ mov(R12, AsmOperand(R10, lsl, lsl_shift), ne); |
|
1644 |
||
1645 |
__ tst(count, 2); |
|
1646 |
if (lsr_shift != 24) { |
|
1647 |
// avoid useless reading R10 when we already have 3 bytes ready in R12 |
|
1648 |
__ ldr(R10, Address(end_from, -4, pre_indexed), ne); |
|
1649 |
__ orr(R12, R12, AsmOperand(R10, lsr,lsr_shift), ne); |
|
1650 |
} |
|
1651 |
||
1652 |
// Note: R12 contains enough bytes ready to write (3 needed at most) |
|
1653 |
// write the 2 MSBs |
|
1654 |
__ mov(R9, AsmOperand(R12, lsr, 16), ne); |
|
1655 |
__ strh(R9, Address(end_to, -2, pre_indexed), ne); |
|
1656 |
// promote remaining to MSB |
|
1657 |
__ mov(R12, AsmOperand(R12, lsl, 16), ne); |
|
1658 |
||
1659 |
__ tst(count, 1); |
|
1660 |
// write the MSB of R12 |
|
1661 |
__ mov(R12, AsmOperand(R12, lsr, 24), ne); |
|
1662 |
__ strb(R12, Address(end_to, -1, pre_indexed), ne); |
|
1663 |
||
1664 |
break; |
|
1665 |
} |
|
1666 |
||
1667 |
__ BIND(L_done); |
|
1668 |
return 0; // no minimum |
|
1669 |
} |
|
1670 |
||
1671 |
// This method is very useful for merging forward/backward implementations |
|
1672 |
Address get_addr_with_indexing(Register base, int delta, bool forward) { |
|
1673 |
if (forward) { |
|
1674 |
return Address(base, delta, post_indexed); |
|
1675 |
} else { |
|
1676 |
return Address(base, -delta, pre_indexed); |
|
1677 |
} |
|
1678 |
} |
|
1679 |
||
1680 |
void load_one(Register rd, Register from, int size_in_bytes, bool forward, AsmCondition cond = al, Register rd2 = noreg) { |
|
1681 |
assert_different_registers(from, rd, rd2); |
|
1682 |
if (size_in_bytes < 8) { |
|
1683 |
Address addr = get_addr_with_indexing(from, size_in_bytes, forward); |
|
1684 |
__ load_sized_value(rd, addr, size_in_bytes, false, cond); |
|
1685 |
} else { |
|
1686 |
assert (rd2 != noreg, "second value register must be specified"); |
|
1687 |
assert (rd->encoding() < rd2->encoding(), "wrong value register set"); |
|
1688 |
||
1689 |
if (forward) { |
|
1690 |
__ ldmia(from, RegisterSet(rd) | rd2, writeback, cond); |
|
1691 |
} else { |
|
1692 |
__ ldmdb(from, RegisterSet(rd) | rd2, writeback, cond); |
|
1693 |
} |
|
1694 |
} |
|
1695 |
} |
|
1696 |
||
1697 |
void store_one(Register rd, Register to, int size_in_bytes, bool forward, AsmCondition cond = al, Register rd2 = noreg) { |
|
1698 |
assert_different_registers(to, rd, rd2); |
|
1699 |
if (size_in_bytes < 8) { |
|
1700 |
Address addr = get_addr_with_indexing(to, size_in_bytes, forward); |
|
1701 |
__ store_sized_value(rd, addr, size_in_bytes, cond); |
|
1702 |
} else { |
|
1703 |
assert (rd2 != noreg, "second value register must be specified"); |
|
1704 |
assert (rd->encoding() < rd2->encoding(), "wrong value register set"); |
|
1705 |
||
1706 |
if (forward) { |
|
1707 |
__ stmia(to, RegisterSet(rd) | rd2, writeback, cond); |
|
1708 |
} else { |
|
1709 |
__ stmdb(to, RegisterSet(rd) | rd2, writeback, cond); |
|
1710 |
} |
|
1711 |
} |
|
1712 |
} |
|
1713 |
||
1714 |
// Copies data from 'from' to 'to' in specified direction to align 'from' by 64 bits. |
|
1715 |
// (on 32-bit ARM 64-bit alignment is better for LDM). |
|
1716 |
// |
|
1717 |
// Arguments: |
|
1718 |
// from: beginning (if forward) or upper bound (if !forward) of the region to be read |
|
1719 |
// to: beginning (if forward) or upper bound (if !forward) of the region to be written |
|
1720 |
// count: 32-bit int, maximum number of elements which can be copied |
|
1721 |
// bytes_per_count: size of an element |
|
1722 |
// forward: specifies copy direction |
|
1723 |
// |
|
1724 |
// Notes: |
|
1725 |
// 'from' and 'to' must be aligned by 'bytes_per_count' |
|
1726 |
// 'count' must not be less than the returned value |
|
1727 |
// shifts 'from' and 'to' by the number of copied bytes in corresponding direction |
|
1728 |
// decreases 'count' by the number of elements copied |
|
1729 |
// |
|
1730 |
// Returns maximum number of bytes which may be copied. |
|
1731 |
int align_src(Register from, Register to, Register count, Register tmp, int bytes_per_count, bool forward) { |
|
1732 |
assert_different_registers(from, to, count, tmp); |
|
1733 |
if (bytes_per_count < 8) { |
|
1734 |
Label L_align_src; |
|
1735 |
__ BIND(L_align_src); |
|
1736 |
__ tst(from, 7); |
|
1737 |
// ne => not aligned: copy one element and (if bytes_per_count < 4) loop |
|
1738 |
__ sub(count, count, 1, ne); |
|
1739 |
load_one(tmp, from, bytes_per_count, forward, ne); |
|
1740 |
store_one(tmp, to, bytes_per_count, forward, ne); |
|
1741 |
if (bytes_per_count < 4) { |
|
1742 |
__ b(L_align_src, ne); // if bytes_per_count == 4, then 0 or 1 loop iterations are enough |
|
1743 |
} |
|
1744 |
} |
|
1745 |
return 7/bytes_per_count; |
|
1746 |
} |
|
1747 |
||
1748 |
// Copies 'count' of 'bytes_per_count'-sized elements in the specified direction. |
|
1749 |
// |
|
1750 |
// Arguments: |
|
1751 |
// from: beginning (if forward) or upper bound (if !forward) of the region to be read |
|
1752 |
// to: beginning (if forward) or upper bound (if !forward) of the region to be written |
|
1753 |
// count: 32-bit int, number of elements to be copied |
|
1754 |
// entry: copy loop entry point |
|
1755 |
// bytes_per_count: size of an element |
|
1756 |
// forward: specifies copy direction |
|
1757 |
// |
|
1758 |
// Notes: |
|
1759 |
// shifts 'from' and 'to' |
|
55490
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1760 |
void copy_small_array(Register from, Register to, Register count, Register tmp, Register tmp2, int bytes_per_count, bool forward, Label & entry, bool unsafe_copy = false) { |
42664 | 1761 |
assert_different_registers(from, to, count, tmp); |
1762 |
||
55490
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1763 |
{ |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1764 |
// UnsafeCopyMemory page error: continue after ucm |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1765 |
UnsafeCopyMemoryMark ucmm(this, unsafe_copy, true); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1766 |
__ align(OptoLoopAlignment); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1767 |
Label L_small_loop; |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1768 |
__ BIND(L_small_loop); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1769 |
store_one(tmp, to, bytes_per_count, forward, al, tmp2); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1770 |
__ BIND(entry); // entry point |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1771 |
__ subs(count, count, 1); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1772 |
load_one(tmp, from, bytes_per_count, forward, ge, tmp2); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1773 |
__ b(L_small_loop, ge); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1774 |
} |
42664 | 1775 |
} |
1776 |
||
1777 |
// Aligns 'to' by reading one word from 'from' and writting its part to 'to'. |
|
1778 |
// |
|
1779 |
// Arguments: |
|
1780 |
// to: beginning (if forward) or upper bound (if !forward) of the region to be written |
|
1781 |
// count: 32-bit int, number of elements allowed to be copied |
|
1782 |
// to_remainder: remainder of dividing 'to' by wordSize |
|
1783 |
// bytes_per_count: size of an element |
|
1784 |
// forward: specifies copy direction |
|
1785 |
// Rval: contains an already read but not yet written word; |
|
1786 |
// its' LSBs (if forward) or MSBs (if !forward) are to be written to align 'to'. |
|
1787 |
// |
|
1788 |
// Notes: |
|
1789 |
// 'count' must not be less then the returned value |
|
1790 |
// 'to' must be aligned by bytes_per_count but must not be aligned by wordSize |
|
1791 |
// shifts 'to' by the number of written bytes (so that it becomes the bound of memory to be written) |
|
1792 |
// decreases 'count' by the the number of elements written |
|
1793 |
// Rval's MSBs or LSBs remain to be written further by generate_{forward,backward}_shifted_copy_loop |
|
1794 |
int align_dst(Register to, Register count, Register Rval, Register tmp, |
|
1795 |
int to_remainder, int bytes_per_count, bool forward) { |
|
1796 |
assert_different_registers(to, count, tmp, Rval); |
|
1797 |
||
1798 |
assert (0 < to_remainder && to_remainder < wordSize, "to_remainder is not valid"); |
|
1799 |
assert (to_remainder % bytes_per_count == 0, "to must be aligned by bytes_per_count"); |
|
1800 |
||
1801 |
int bytes_to_write = forward ? (wordSize - to_remainder) : to_remainder; |
|
1802 |
||
1803 |
int offset = 0; |
|
1804 |
||
1805 |
for (int l = 0; l < LogBytesPerWord; ++l) { |
|
1806 |
int s = (1 << l); |
|
1807 |
if (bytes_to_write & s) { |
|
1808 |
int new_offset = offset + s*BitsPerByte; |
|
1809 |
if (forward) { |
|
1810 |
if (offset == 0) { |
|
1811 |
store_one(Rval, to, s, forward); |
|
1812 |
} else { |
|
1813 |
__ logical_shift_right(tmp, Rval, offset); |
|
1814 |
store_one(tmp, to, s, forward); |
|
1815 |
} |
|
1816 |
} else { |
|
1817 |
__ logical_shift_right(tmp, Rval, BitsPerWord - new_offset); |
|
1818 |
store_one(tmp, to, s, forward); |
|
1819 |
} |
|
1820 |
||
1821 |
offset = new_offset; |
|
1822 |
} |
|
1823 |
} |
|
1824 |
||
1825 |
assert (offset == bytes_to_write * BitsPerByte, "all bytes must be copied"); |
|
1826 |
||
1827 |
__ sub_32(count, count, bytes_to_write/bytes_per_count); |
|
1828 |
||
1829 |
return bytes_to_write / bytes_per_count; |
|
1830 |
} |
|
1831 |
||
1832 |
// Copies 'count' of elements using shifted copy loop |
|
1833 |
// |
|
1834 |
// Arguments: |
|
1835 |
// from: beginning (if forward) or upper bound (if !forward) of the region to be read |
|
1836 |
// to: beginning (if forward) or upper bound (if !forward) of the region to be written |
|
1837 |
// count: 32-bit int, number of elements to be copied |
|
1838 |
// to_remainder: remainder of dividing 'to' by wordSize |
|
1839 |
// bytes_per_count: size of an element |
|
1840 |
// forward: specifies copy direction |
|
1841 |
// Rval: contains an already read but not yet written word |
|
1842 |
// |
|
1843 |
// |
|
1844 |
// Notes: |
|
1845 |
// 'count' must not be less then the returned value |
|
1846 |
// 'from' must be aligned by wordSize |
|
1847 |
// 'to' must be aligned by bytes_per_count but must not be aligned by wordSize |
|
1848 |
// shifts 'to' by the number of copied bytes |
|
1849 |
// |
|
1850 |
// Scratches R3-R10, R12 |
|
1851 |
int align_dst_and_generate_shifted_copy_loop(Register from, Register to, Register count, Register Rval, |
|
1852 |
int to_remainder, int bytes_per_count, bool forward) { |
|
1853 |
||
1854 |
assert (0 < to_remainder && to_remainder < wordSize, "to_remainder is invalid"); |
|
1855 |
||
52351 | 1856 |
const Register tmp = forward ? R3 : R12; |
42664 | 1857 |
assert_different_registers(from, to, count, Rval, tmp); |
1858 |
||
1859 |
int required_to_align = align_dst(to, count, Rval, tmp, to_remainder, bytes_per_count, forward); |
|
1860 |
||
1861 |
int lsr_shift = (wordSize - to_remainder) * BitsPerByte; |
|
1862 |
int lsl_shift = to_remainder * BitsPerByte; |
|
1863 |
||
1864 |
int min_copy; |
|
1865 |
if (forward) { |
|
1866 |
min_copy = generate_forward_shifted_copy_loop(from, to, count, bytes_per_count, lsr_shift, lsl_shift); |
|
1867 |
} else { |
|
1868 |
min_copy = generate_backward_shifted_copy_loop(from, to, count, bytes_per_count, lsr_shift, lsl_shift); |
|
1869 |
} |
|
1870 |
||
1871 |
return min_copy + required_to_align; |
|
1872 |
} |
|
1873 |
||
1874 |
// Copies 'count' of elements using shifted copy loop |
|
1875 |
// |
|
1876 |
// Arguments: |
|
1877 |
// from: beginning (if forward) or upper bound (if !forward) of the region to be read |
|
1878 |
// to: beginning (if forward) or upper bound (if !forward) of the region to be written |
|
1879 |
// count: 32-bit int, number of elements to be copied |
|
1880 |
// bytes_per_count: size of an element |
|
1881 |
// forward: specifies copy direction |
|
1882 |
// |
|
1883 |
// Notes: |
|
1884 |
// 'count' must not be less then the returned value |
|
1885 |
// 'from' must be aligned by wordSize |
|
1886 |
// 'to' must be aligned by bytes_per_count but must not be aligned by wordSize |
|
1887 |
// shifts 'to' by the number of copied bytes |
|
1888 |
// |
|
1889 |
// Scratches 'from', 'count', R3 and R12. |
|
52351 | 1890 |
// R4-R10 saved for use. |
55490
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1891 |
int align_dst_and_generate_shifted_copy_loop(Register from, Register to, Register count, int bytes_per_count, bool forward, bool unsafe_copy = false) { |
42664 | 1892 |
|
1893 |
const Register Rval = forward ? R12 : R3; // as generate_{forward,backward}_shifted_copy_loop expect |
|
1894 |
||
1895 |
int min_copy = 0; |
|
1896 |
||
1897 |
// Note: if {seq} is a sequence of numbers, L{seq} means that if the execution reaches this point, |
|
1898 |
// then the remainder of 'to' divided by wordSize is one of elements of {seq}. |
|
1899 |
||
1900 |
__ push(RegisterSet(R4,R10)); |
|
55490
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1901 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1902 |
{ |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1903 |
// UnsafeCopyMemory page error: continue after ucm |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1904 |
UnsafeCopyMemoryMark ucmm(this, unsafe_copy, true); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1905 |
load_one(Rval, from, wordSize, forward); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1906 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1907 |
switch (bytes_per_count) { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1908 |
case 2: |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1909 |
min_copy = align_dst_and_generate_shifted_copy_loop(from, to, count, Rval, 2, bytes_per_count, forward); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1910 |
break; |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1911 |
case 1: |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1912 |
{ |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1913 |
Label L1, L2, L3; |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1914 |
int min_copy1, min_copy2, min_copy3; |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1915 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1916 |
Label L_loop_finished; |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1917 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1918 |
if (forward) { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1919 |
__ tbz(to, 0, L2); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1920 |
__ tbz(to, 1, L1); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1921 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1922 |
__ BIND(L3); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1923 |
min_copy3 = align_dst_and_generate_shifted_copy_loop(from, to, count, Rval, 3, bytes_per_count, forward); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1924 |
__ b(L_loop_finished); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1925 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1926 |
__ BIND(L1); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1927 |
min_copy1 = align_dst_and_generate_shifted_copy_loop(from, to, count, Rval, 1, bytes_per_count, forward); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1928 |
__ b(L_loop_finished); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1929 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1930 |
__ BIND(L2); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1931 |
min_copy2 = align_dst_and_generate_shifted_copy_loop(from, to, count, Rval, 2, bytes_per_count, forward); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1932 |
} else { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1933 |
__ tbz(to, 0, L2); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1934 |
__ tbnz(to, 1, L3); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1935 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1936 |
__ BIND(L1); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1937 |
min_copy1 = align_dst_and_generate_shifted_copy_loop(from, to, count, Rval, 1, bytes_per_count, forward); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1938 |
__ b(L_loop_finished); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1939 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1940 |
__ BIND(L3); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1941 |
min_copy3 = align_dst_and_generate_shifted_copy_loop(from, to, count, Rval, 3, bytes_per_count, forward); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1942 |
__ b(L_loop_finished); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1943 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1944 |
__ BIND(L2); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1945 |
min_copy2 = align_dst_and_generate_shifted_copy_loop(from, to, count, Rval, 2, bytes_per_count, forward); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1946 |
} |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1947 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1948 |
min_copy = MAX2(MAX2(min_copy1, min_copy2), min_copy3); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1949 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1950 |
__ BIND(L_loop_finished); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1951 |
|
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1952 |
break; |
42664 | 1953 |
} |
55490
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1954 |
default: |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1955 |
ShouldNotReachHere(); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1956 |
break; |
42664 | 1957 |
} |
1958 |
} |
|
1959 |
__ pop(RegisterSet(R4,R10)); |
|
1960 |
||
1961 |
return min_copy; |
|
1962 |
} |
|
1963 |
||
1964 |
#ifndef PRODUCT |
|
1965 |
int * get_arraycopy_counter(int bytes_per_count) { |
|
1966 |
switch (bytes_per_count) { |
|
1967 |
case 1: |
|
1968 |
return &SharedRuntime::_jbyte_array_copy_ctr; |
|
1969 |
case 2: |
|
1970 |
return &SharedRuntime::_jshort_array_copy_ctr; |
|
1971 |
case 4: |
|
1972 |
return &SharedRuntime::_jint_array_copy_ctr; |
|
1973 |
case 8: |
|
1974 |
return &SharedRuntime::_jlong_array_copy_ctr; |
|
1975 |
default: |
|
1976 |
ShouldNotReachHere(); |
|
1977 |
return NULL; |
|
1978 |
} |
|
1979 |
} |
|
1980 |
#endif // !PRODUCT |
|
1981 |
||
55490
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1982 |
address generate_unsafecopy_common_error_exit() { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1983 |
address start_pc = __ pc(); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1984 |
__ mov(R0, 0); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1985 |
__ ret(); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1986 |
return start_pc; |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1987 |
} |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
1988 |
|
42664 | 1989 |
// |
1990 |
// Generate stub for primitive array copy. If "aligned" is true, the |
|
1991 |
// "from" and "to" addresses are assumed to be heapword aligned. |
|
1992 |
// |
|
1993 |
// If "disjoint" is true, arrays are assumed to be disjoint, otherwise they may overlap and |
|
1994 |
// "nooverlap_target" must be specified as the address to jump if they don't. |
|
1995 |
// |
|
1996 |
// Arguments for generated stub: |
|
1997 |
// from: R0 |
|
1998 |
// to: R1 |
|
1999 |
// count: R2 treated as signed 32-bit int |
|
2000 |
// |
|
2001 |
address generate_primitive_copy(bool aligned, const char * name, bool status, int bytes_per_count, bool disjoint, address nooverlap_target = NULL) { |
|
2002 |
__ align(CodeEntryAlignment); |
|
2003 |
StubCodeMark mark(this, "StubRoutines", name); |
|
2004 |
address start = __ pc(); |
|
2005 |
||
2006 |
const Register from = R0; // source array address |
|
2007 |
const Register to = R1; // destination array address |
|
2008 |
const Register count = R2; // elements count |
|
2009 |
const Register tmp1 = R3; |
|
2010 |
const Register tmp2 = R12; |
|
2011 |
||
2012 |
if (!aligned) { |
|
2013 |
BLOCK_COMMENT("Entry:"); |
|
2014 |
} |
|
2015 |
||
2016 |
__ zap_high_non_significant_bits(R2); |
|
2017 |
||
2018 |
if (!disjoint) { |
|
2019 |
assert (nooverlap_target != NULL, "must be specified for conjoint case"); |
|
2020 |
array_overlap_test(nooverlap_target, exact_log2(bytes_per_count), tmp1, tmp2); |
|
2021 |
} |
|
2022 |
||
2023 |
inc_counter_np(*get_arraycopy_counter(bytes_per_count), tmp1, tmp2); |
|
2024 |
||
2025 |
// Conjoint case: since execution reaches this point, the arrays overlap, so performing backward copy |
|
2026 |
// Disjoint case: perform forward copy |
|
2027 |
bool forward = disjoint; |
|
2028 |
||
2029 |
||
2030 |
if (!forward) { |
|
2031 |
// Set 'from' and 'to' to upper bounds |
|
2032 |
int log_bytes_per_count = exact_log2(bytes_per_count); |
|
2033 |
__ add_ptr_scaled_int32(to, to, count, log_bytes_per_count); |
|
2034 |
__ add_ptr_scaled_int32(from, from, count, log_bytes_per_count); |
|
2035 |
} |
|
2036 |
||
2037 |
// There are two main copy loop implementations: |
|
2038 |
// *) The huge and complex one applicable only for large enough arrays |
|
2039 |
// *) The small and simple one applicable for any array (but not efficient for large arrays). |
|
2040 |
// Currently "small" implementation is used if and only if the "large" one could not be used. |
|
2041 |
// XXX optim: tune the limit higher ? |
|
2042 |
// Large implementation lower applicability bound is actually determined by |
|
2043 |
// aligned copy loop which require <=7 bytes for src alignment, and 8 words for aligned copy loop. |
|
2044 |
const int small_copy_limit = (8*wordSize + 7) / bytes_per_count; |
|
2045 |
||
2046 |
Label L_small_array; |
|
2047 |
__ cmp_32(count, small_copy_limit); |
|
52351 | 2048 |
__ b(L_small_array, le); |
42664 | 2049 |
|
2050 |
// Otherwise proceed with large implementation. |
|
2051 |
||
2052 |
bool from_is_aligned = (bytes_per_count >= 8); |
|
2053 |
if (aligned && forward && (HeapWordSize % 8 == 0)) { |
|
2054 |
// if 'from' is heapword aligned and HeapWordSize is divisible by 8, |
|
2055 |
// then from is aligned by 8 |
|
2056 |
from_is_aligned = true; |
|
2057 |
} |
|
2058 |
||
55490
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
2059 |
int count_required_to_align = 0; |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
2060 |
{ |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
2061 |
// UnsafeCopyMemoryMark page error: continue at UnsafeCopyMemory common_error_exit |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
2062 |
UnsafeCopyMemoryMark ucmm(this, !aligned, false); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
2063 |
count_required_to_align = from_is_aligned ? 0 : align_src(from, to, count, tmp1, bytes_per_count, forward); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
2064 |
assert (small_copy_limit >= count_required_to_align, "alignment could exhaust count"); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
2065 |
} |
42664 | 2066 |
|
2067 |
// now 'from' is aligned |
|
2068 |
||
2069 |
bool to_is_aligned = false; |
|
2070 |
||
2071 |
if (bytes_per_count >= wordSize) { |
|
2072 |
// 'to' is aligned by bytes_per_count, so it is aligned by wordSize |
|
2073 |
to_is_aligned = true; |
|
2074 |
} else { |
|
2075 |
if (aligned && (8 % HeapWordSize == 0) && (HeapWordSize % wordSize == 0)) { |
|
2076 |
// Originally 'from' and 'to' were heapword aligned; |
|
2077 |
// (from - to) has not been changed, so since now 'from' is 8-byte aligned, then it is also heapword aligned, |
|
2078 |
// so 'to' is also heapword aligned and thus aligned by wordSize. |
|
2079 |
to_is_aligned = true; |
|
2080 |
} |
|
2081 |
} |
|
2082 |
||
2083 |
Label L_unaligned_dst; |
|
2084 |
||
2085 |
if (!to_is_aligned) { |
|
2086 |
BLOCK_COMMENT("Check dst alignment:"); |
|
2087 |
__ tst(to, wordSize - 1); |
|
2088 |
__ b(L_unaligned_dst, ne); // 'to' is not aligned |
|
2089 |
} |
|
2090 |
||
2091 |
// 'from' and 'to' are properly aligned |
|
2092 |
||
2093 |
int min_copy; |
|
2094 |
if (forward) { |
|
55490
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
2095 |
min_copy = generate_forward_aligned_copy_loop(from, to, count, bytes_per_count, !aligned /*add UnsafeCopyMemory entry*/); |
42664 | 2096 |
} else { |
55490
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
2097 |
min_copy = generate_backward_aligned_copy_loop(from, to, count, bytes_per_count, !aligned /*add UnsafeCopyMemory entry*/); |
42664 | 2098 |
} |
2099 |
assert(small_copy_limit >= count_required_to_align + min_copy, "first loop might exhaust count"); |
|
2100 |
||
2101 |
if (status) { |
|
2102 |
__ mov(R0, 0); // OK |
|
2103 |
} |
|
2104 |
||
2105 |
__ ret(); |
|
2106 |
||
2107 |
{ |
|
55490
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
2108 |
copy_small_array(from, to, count, tmp1, tmp2, bytes_per_count, forward, L_small_array /* entry */, !aligned /*add UnsafeCopyMemory entry*/); |
42664 | 2109 |
|
2110 |
if (status) { |
|
2111 |
__ mov(R0, 0); // OK |
|
2112 |
} |
|
2113 |
||
2114 |
__ ret(); |
|
2115 |
} |
|
2116 |
||
2117 |
if (! to_is_aligned) { |
|
2118 |
__ BIND(L_unaligned_dst); |
|
55490
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
2119 |
int min_copy_shifted = align_dst_and_generate_shifted_copy_loop(from, to, count, bytes_per_count, forward, !aligned /*add UnsafeCopyMemory entry*/); |
42664 | 2120 |
assert (small_copy_limit >= count_required_to_align + min_copy_shifted, "first loop might exhaust count"); |
2121 |
||
2122 |
if (status) { |
|
2123 |
__ mov(R0, 0); // OK |
|
2124 |
} |
|
2125 |
||
2126 |
__ ret(); |
|
2127 |
} |
|
2128 |
||
2129 |
return start; |
|
2130 |
} |
|
2131 |
||
2132 |
||
2133 |
// Generates pattern of code to be placed after raw data copying in generate_oop_copy |
|
2134 |
// Includes return from arraycopy stub. |
|
2135 |
// |
|
2136 |
// Arguments: |
|
2137 |
// to: destination pointer after copying. |
|
2138 |
// if 'forward' then 'to' == upper bound, else 'to' == beginning of the modified region |
|
2139 |
// count: total number of copied elements, 32-bit int |
|
2140 |
// |
|
52351 | 2141 |
// Blows all volatile R0-R3, Rtemp, LR) and 'to', 'count', 'tmp' registers. |
49484
ee8fa73b90f9
8198949: Modularize arraycopy stub routine GC barriers
eosterlund
parents:
49455
diff
changeset
|
2142 |
void oop_arraycopy_stub_epilogue_helper(Register to, Register count, Register tmp, bool status, bool forward, DecoratorSet decorators) { |
42664 | 2143 |
assert_different_registers(to, count, tmp); |
2144 |
||
2145 |
if (forward) { |
|
2146 |
// 'to' is upper bound of the modified region |
|
2147 |
// restore initial dst: |
|
2148 |
__ sub_ptr_scaled_int32(to, to, count, LogBytesPerHeapOop); |
|
2149 |
} |
|
2150 |
||
2151 |
// 'to' is the beginning of the region |
|
2152 |
||
49484
ee8fa73b90f9
8198949: Modularize arraycopy stub routine GC barriers
eosterlund
parents:
49455
diff
changeset
|
2153 |
BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); |
49674
c39f20946b37
8200627: aarch32 - Broken build after JDK-8198949
dpochepk
parents:
49484
diff
changeset
|
2154 |
bs->arraycopy_epilogue(_masm, decorators, true, to, count, tmp); |
42664 | 2155 |
|
2156 |
if (status) { |
|
2157 |
__ mov(R0, 0); // OK |
|
2158 |
} |
|
2159 |
||
2160 |
__ pop(PC); |
|
2161 |
} |
|
2162 |
||
2163 |
||
2164 |
// Generate stub for assign-compatible oop copy. If "aligned" is true, the |
|
2165 |
// "from" and "to" addresses are assumed to be heapword aligned. |
|
2166 |
// |
|
2167 |
// If "disjoint" is true, arrays are assumed to be disjoint, otherwise they may overlap and |
|
2168 |
// "nooverlap_target" must be specified as the address to jump if they don't. |
|
2169 |
// |
|
2170 |
// Arguments for generated stub: |
|
2171 |
// from: R0 |
|
2172 |
// to: R1 |
|
2173 |
// count: R2 treated as signed 32-bit int |
|
2174 |
// |
|
2175 |
address generate_oop_copy(bool aligned, const char * name, bool status, bool disjoint, address nooverlap_target = NULL) { |
|
2176 |
__ align(CodeEntryAlignment); |
|
2177 |
StubCodeMark mark(this, "StubRoutines", name); |
|
2178 |
address start = __ pc(); |
|
2179 |
||
2180 |
Register from = R0; |
|
2181 |
Register to = R1; |
|
2182 |
Register count = R2; |
|
2183 |
Register tmp1 = R3; |
|
2184 |
Register tmp2 = R12; |
|
2185 |
||
2186 |
||
2187 |
if (!aligned) { |
|
2188 |
BLOCK_COMMENT("Entry:"); |
|
2189 |
} |
|
2190 |
||
2191 |
__ zap_high_non_significant_bits(R2); |
|
2192 |
||
2193 |
if (!disjoint) { |
|
2194 |
assert (nooverlap_target != NULL, "must be specified for conjoint case"); |
|
2195 |
array_overlap_test(nooverlap_target, LogBytesPerHeapOop, tmp1, tmp2); |
|
2196 |
} |
|
2197 |
||
2198 |
inc_counter_np(SharedRuntime::_oop_array_copy_ctr, tmp1, tmp2); |
|
2199 |
||
2200 |
// Conjoint case: since execution reaches this point, the arrays overlap, so performing backward copy |
|
2201 |
// Disjoint case: perform forward copy |
|
2202 |
bool forward = disjoint; |
|
2203 |
||
2204 |
const int bytes_per_count = BytesPerHeapOop; |
|
2205 |
const int log_bytes_per_count = LogBytesPerHeapOop; |
|
2206 |
||
2207 |
const Register saved_count = LR; |
|
2208 |
const int callee_saved_regs = 3; // R0-R2 |
|
2209 |
||
2210 |
// LR is used later to save barrier args |
|
2211 |
__ push(LR); |
|
2212 |
||
50728 | 2213 |
DecoratorSet decorators = IN_HEAP | IS_ARRAY; |
49484
ee8fa73b90f9
8198949: Modularize arraycopy stub routine GC barriers
eosterlund
parents:
49455
diff
changeset
|
2214 |
if (disjoint) { |
ee8fa73b90f9
8198949: Modularize arraycopy stub routine GC barriers
eosterlund
parents:
49455
diff
changeset
|
2215 |
decorators |= ARRAYCOPY_DISJOINT; |
ee8fa73b90f9
8198949: Modularize arraycopy stub routine GC barriers
eosterlund
parents:
49455
diff
changeset
|
2216 |
} |
ee8fa73b90f9
8198949: Modularize arraycopy stub routine GC barriers
eosterlund
parents:
49455
diff
changeset
|
2217 |
if (aligned) { |
ee8fa73b90f9
8198949: Modularize arraycopy stub routine GC barriers
eosterlund
parents:
49455
diff
changeset
|
2218 |
decorators |= ARRAYCOPY_ALIGNED; |
ee8fa73b90f9
8198949: Modularize arraycopy stub routine GC barriers
eosterlund
parents:
49455
diff
changeset
|
2219 |
} |
ee8fa73b90f9
8198949: Modularize arraycopy stub routine GC barriers
eosterlund
parents:
49455
diff
changeset
|
2220 |
|
ee8fa73b90f9
8198949: Modularize arraycopy stub routine GC barriers
eosterlund
parents:
49455
diff
changeset
|
2221 |
BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); |
49674
c39f20946b37
8200627: aarch32 - Broken build after JDK-8198949
dpochepk
parents:
49484
diff
changeset
|
2222 |
bs->arraycopy_prologue(_masm, decorators, true, to, count, callee_saved_regs); |
42664 | 2223 |
|
2224 |
// save arguments for barrier generation (after the pre barrier) |
|
2225 |
__ mov(saved_count, count); |
|
2226 |
||
2227 |
if (!forward) { |
|
2228 |
__ add_ptr_scaled_int32(to, to, count, log_bytes_per_count); |
|
2229 |
__ add_ptr_scaled_int32(from, from, count, log_bytes_per_count); |
|
2230 |
} |
|
2231 |
||
2232 |
// for short arrays, just do single element copy |
|
2233 |
Label L_small_array; |
|
2234 |
const int small_copy_limit = (8*wordSize + 7)/bytes_per_count; // XXX optim: tune the limit higher ? |
|
2235 |
__ cmp_32(count, small_copy_limit); |
|
2236 |
__ b(L_small_array, le); |
|
2237 |
||
2238 |
bool from_is_aligned = (bytes_per_count >= 8); |
|
2239 |
if (aligned && forward && (HeapWordSize % 8 == 0)) { |
|
2240 |
// if 'from' is heapword aligned and HeapWordSize is divisible by 8, |
|
2241 |
// then from is aligned by 8 |
|
2242 |
from_is_aligned = true; |
|
2243 |
} |
|
2244 |
||
2245 |
int count_required_to_align = from_is_aligned ? 0 : align_src(from, to, count, tmp1, bytes_per_count, forward); |
|
2246 |
assert (small_copy_limit >= count_required_to_align, "alignment could exhaust count"); |
|
2247 |
||
2248 |
// now 'from' is aligned |
|
2249 |
||
2250 |
bool to_is_aligned = false; |
|
2251 |
||
2252 |
if (bytes_per_count >= wordSize) { |
|
2253 |
// 'to' is aligned by bytes_per_count, so it is aligned by wordSize |
|
2254 |
to_is_aligned = true; |
|
2255 |
} else { |
|
2256 |
if (aligned && (8 % HeapWordSize == 0) && (HeapWordSize % wordSize == 0)) { |
|
2257 |
// Originally 'from' and 'to' were heapword aligned; |
|
2258 |
// (from - to) has not been changed, so since now 'from' is 8-byte aligned, then it is also heapword aligned, |
|
2259 |
// so 'to' is also heapword aligned and thus aligned by wordSize. |
|
2260 |
to_is_aligned = true; |
|
2261 |
} |
|
2262 |
} |
|
2263 |
||
2264 |
Label L_unaligned_dst; |
|
2265 |
||
2266 |
if (!to_is_aligned) { |
|
2267 |
BLOCK_COMMENT("Check dst alignment:"); |
|
2268 |
__ tst(to, wordSize - 1); |
|
2269 |
__ b(L_unaligned_dst, ne); // 'to' is not aligned |
|
2270 |
} |
|
2271 |
||
2272 |
int min_copy; |
|
2273 |
if (forward) { |
|
2274 |
min_copy = generate_forward_aligned_copy_loop(from, to, count, bytes_per_count); |
|
2275 |
} else { |
|
2276 |
min_copy = generate_backward_aligned_copy_loop(from, to, count, bytes_per_count); |
|
2277 |
} |
|
2278 |
assert(small_copy_limit >= count_required_to_align + min_copy, "first loop might exhaust count"); |
|
2279 |
||
49484
ee8fa73b90f9
8198949: Modularize arraycopy stub routine GC barriers
eosterlund
parents:
49455
diff
changeset
|
2280 |
oop_arraycopy_stub_epilogue_helper(to, saved_count, /* tmp */ tmp1, status, forward, decorators); |
42664 | 2281 |
|
2282 |
{ |
|
2283 |
copy_small_array(from, to, count, tmp1, noreg, bytes_per_count, forward, L_small_array); |
|
2284 |
||
49484
ee8fa73b90f9
8198949: Modularize arraycopy stub routine GC barriers
eosterlund
parents:
49455
diff
changeset
|
2285 |
oop_arraycopy_stub_epilogue_helper(to, saved_count, /* tmp */ tmp1, status, forward, decorators); |
42664 | 2286 |
} |
2287 |
||
2288 |
if (!to_is_aligned) { |
|
2289 |
__ BIND(L_unaligned_dst); |
|
2290 |
ShouldNotReachHere(); |
|
2291 |
int min_copy_shifted = align_dst_and_generate_shifted_copy_loop(from, to, count, bytes_per_count, forward); |
|
2292 |
assert (small_copy_limit >= count_required_to_align + min_copy_shifted, "first loop might exhaust count"); |
|
2293 |
||
49484
ee8fa73b90f9
8198949: Modularize arraycopy stub routine GC barriers
eosterlund
parents:
49455
diff
changeset
|
2294 |
oop_arraycopy_stub_epilogue_helper(to, saved_count, /* tmp */ tmp1, status, forward, decorators); |
42664 | 2295 |
} |
2296 |
||
2297 |
return start; |
|
2298 |
} |
|
2299 |
||
2300 |
// Generate 'unsafe' array copy stub |
|
2301 |
// Though just as safe as the other stubs, it takes an unscaled |
|
2302 |
// size_t argument instead of an element count. |
|
2303 |
// |
|
2304 |
// Arguments for generated stub: |
|
2305 |
// from: R0 |
|
2306 |
// to: R1 |
|
2307 |
// count: R2 byte count, treated as ssize_t, can be zero |
|
2308 |
// |
|
2309 |
// Examines the alignment of the operands and dispatches |
|
2310 |
// to a long, int, short, or byte copy loop. |
|
2311 |
// |
|
2312 |
address generate_unsafe_copy(const char* name) { |
|
2313 |
||
2314 |
const Register R0_from = R0; // source array address |
|
2315 |
const Register R1_to = R1; // destination array address |
|
2316 |
const Register R2_count = R2; // elements count |
|
2317 |
||
2318 |
const Register R3_bits = R3; // test copy of low bits |
|
2319 |
||
2320 |
__ align(CodeEntryAlignment); |
|
2321 |
StubCodeMark mark(this, "StubRoutines", name); |
|
2322 |
address start = __ pc(); |
|
2323 |
const Register tmp = Rtemp; |
|
2324 |
||
2325 |
// bump this on entry, not on exit: |
|
2326 |
inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr, R3, tmp); |
|
2327 |
||
2328 |
__ orr(R3_bits, R0_from, R1_to); |
|
2329 |
__ orr(R3_bits, R2_count, R3_bits); |
|
2330 |
||
2331 |
__ tst(R3_bits, BytesPerLong-1); |
|
2332 |
__ mov(R2_count,AsmOperand(R2_count,asr,LogBytesPerLong), eq); |
|
2333 |
__ jump(StubRoutines::_jlong_arraycopy, relocInfo::runtime_call_type, tmp, eq); |
|
2334 |
||
2335 |
__ tst(R3_bits, BytesPerInt-1); |
|
2336 |
__ mov(R2_count,AsmOperand(R2_count,asr,LogBytesPerInt), eq); |
|
2337 |
__ jump(StubRoutines::_jint_arraycopy, relocInfo::runtime_call_type, tmp, eq); |
|
2338 |
||
2339 |
__ tst(R3_bits, BytesPerShort-1); |
|
2340 |
__ mov(R2_count,AsmOperand(R2_count,asr,LogBytesPerShort), eq); |
|
2341 |
__ jump(StubRoutines::_jshort_arraycopy, relocInfo::runtime_call_type, tmp, eq); |
|
2342 |
||
2343 |
__ jump(StubRoutines::_jbyte_arraycopy, relocInfo::runtime_call_type, tmp); |
|
2344 |
return start; |
|
2345 |
} |
|
2346 |
||
2347 |
// Helper for generating a dynamic type check. |
|
2348 |
// Smashes only the given temp registers. |
|
2349 |
void generate_type_check(Register sub_klass, |
|
2350 |
Register super_check_offset, |
|
2351 |
Register super_klass, |
|
2352 |
Register tmp1, |
|
2353 |
Register tmp2, |
|
2354 |
Register tmp3, |
|
2355 |
Label& L_success) { |
|
2356 |
assert_different_registers(sub_klass, super_check_offset, super_klass, tmp1, tmp2, tmp3); |
|
2357 |
||
2358 |
BLOCK_COMMENT("type_check:"); |
|
2359 |
||
2360 |
// If the pointers are equal, we are done (e.g., String[] elements). |
|
2361 |
||
2362 |
__ cmp(super_klass, sub_klass); |
|
2363 |
__ b(L_success, eq); // fast success |
|
2364 |
||
2365 |
||
2366 |
Label L_loop, L_fail; |
|
2367 |
||
2368 |
int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); |
|
2369 |
||
2370 |
// Check the supertype display: |
|
2371 |
__ ldr(tmp1, Address(sub_klass, super_check_offset)); |
|
2372 |
__ cmp(tmp1, super_klass); |
|
2373 |
__ b(L_success, eq); |
|
2374 |
||
2375 |
__ cmp(super_check_offset, sc_offset); |
|
2376 |
__ b(L_fail, ne); // failure |
|
2377 |
||
2378 |
BLOCK_COMMENT("type_check_slow_path:"); |
|
2379 |
||
2380 |
// a couple of useful fields in sub_klass: |
|
2381 |
int ss_offset = in_bytes(Klass::secondary_supers_offset()); |
|
2382 |
||
2383 |
// Do a linear scan of the secondary super-klass chain. |
|
2384 |
||
2385 |
#ifndef PRODUCT |
|
2386 |
int* pst_counter = &SharedRuntime::_partial_subtype_ctr; |
|
2387 |
__ inc_counter((address) pst_counter, tmp1, tmp2); |
|
2388 |
#endif |
|
2389 |
||
2390 |
Register scan_temp = tmp1; |
|
2391 |
Register count_temp = tmp2; |
|
2392 |
||
2393 |
// We will consult the secondary-super array. |
|
2394 |
__ ldr(scan_temp, Address(sub_klass, ss_offset)); |
|
2395 |
||
2396 |
Register search_key = super_klass; |
|
2397 |
||
2398 |
// Load the array length. |
|
2399 |
__ ldr_s32(count_temp, Address(scan_temp, Array<Klass*>::length_offset_in_bytes())); |
|
2400 |
__ add(scan_temp, scan_temp, Array<Klass*>::base_offset_in_bytes()); |
|
2401 |
||
2402 |
__ add(count_temp, count_temp, 1); |
|
2403 |
||
2404 |
// Top of search loop |
|
2405 |
__ bind(L_loop); |
|
2406 |
// Notes: |
|
2407 |
// scan_temp starts at the array elements |
|
2408 |
// count_temp is 1+size |
|
2409 |
||
2410 |
__ subs(count_temp, count_temp, 1); |
|
2411 |
__ b(L_fail, eq); // not found |
|
2412 |
||
2413 |
// Load next super to check |
|
2414 |
// In the array of super classes elements are pointer sized. |
|
2415 |
int element_size = wordSize; |
|
2416 |
__ ldr(tmp3, Address(scan_temp, element_size, post_indexed)); |
|
2417 |
||
2418 |
// Look for Rsuper_klass on Rsub_klass's secondary super-class-overflow list |
|
2419 |
__ cmp(tmp3, search_key); |
|
2420 |
||
2421 |
// A miss means we are NOT a subtype and need to keep looping |
|
2422 |
__ b(L_loop, ne); |
|
2423 |
||
2424 |
// Falling out the bottom means we found a hit; we ARE a subtype |
|
2425 |
||
2426 |
// Success. Cache the super we found and proceed in triumph. |
|
2427 |
__ str(super_klass, Address(sub_klass, sc_offset)); |
|
2428 |
||
2429 |
// Jump to success |
|
2430 |
__ b(L_success); |
|
2431 |
||
2432 |
// Fall through on failure! |
|
2433 |
__ bind(L_fail); |
|
2434 |
} |
|
2435 |
||
2436 |
// Generate stub for checked oop copy. |
|
2437 |
// |
|
2438 |
// Arguments for generated stub: |
|
2439 |
// from: R0 |
|
2440 |
// to: R1 |
|
2441 |
// count: R2 treated as signed 32-bit int |
|
2442 |
// ckoff: R3 (super_check_offset) |
|
52351 | 2443 |
// ckval: R4 (super_klass) |
42664 | 2444 |
// ret: R0 zero for success; (-1^K) where K is partial transfer count (32-bit) |
2445 |
// |
|
2446 |
address generate_checkcast_copy(const char * name) { |
|
2447 |
__ align(CodeEntryAlignment); |
|
2448 |
StubCodeMark mark(this, "StubRoutines", name); |
|
2449 |
address start = __ pc(); |
|
2450 |
||
2451 |
const Register from = R0; // source array address |
|
2452 |
const Register to = R1; // destination array address |
|
2453 |
const Register count = R2; // elements count |
|
2454 |
||
2455 |
const Register R3_ckoff = R3; // super_check_offset |
|
2456 |
const Register R4_ckval = R4; // super_klass |
|
2457 |
||
52351 | 2458 |
const int callee_saved_regs = 4; // LR saved differently |
42664 | 2459 |
|
49484
ee8fa73b90f9
8198949: Modularize arraycopy stub routine GC barriers
eosterlund
parents:
49455
diff
changeset
|
2460 |
Label load_element, store_element, do_epilogue, fail; |
42664 | 2461 |
|
2462 |
BLOCK_COMMENT("Entry:"); |
|
2463 |
||
2464 |
__ zap_high_non_significant_bits(R2); |
|
2465 |
||
2466 |
int pushed = 0; |
|
2467 |
__ push(LR); |
|
2468 |
pushed+=1; |
|
2469 |
||
50728 | 2470 |
DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_CHECKCAST; |
49484
ee8fa73b90f9
8198949: Modularize arraycopy stub routine GC barriers
eosterlund
parents:
49455
diff
changeset
|
2471 |
|
ee8fa73b90f9
8198949: Modularize arraycopy stub routine GC barriers
eosterlund
parents:
49455
diff
changeset
|
2472 |
BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); |
49674
c39f20946b37
8200627: aarch32 - Broken build after JDK-8198949
dpochepk
parents:
49484
diff
changeset
|
2473 |
bs->arraycopy_prologue(_masm, decorators, true, to, count, callee_saved_regs); |
42664 | 2474 |
|
2475 |
const RegisterSet caller_saved_regs = RegisterSet(R4,R6) | RegisterSet(R8,R9) | altFP_7_11; |
|
2476 |
__ push(caller_saved_regs); |
|
2477 |
assert(caller_saved_regs.size() == 6, "check the count"); |
|
2478 |
pushed+=6; |
|
2479 |
||
2480 |
__ ldr(R4_ckval,Address(SP, wordSize*pushed)); // read the argument that was on the stack |
|
2481 |
||
2482 |
// Save arguments for barrier generation (after the pre barrier): |
|
2483 |
// - must be a caller saved register and not LR |
|
2484 |
// - ARM32: avoid R10 in case RThread is needed |
|
52351 | 2485 |
const Register saved_count = altFP_7_11; |
42664 | 2486 |
__ movs(saved_count, count); // and test count |
2487 |
__ b(load_element,ne); |
|
2488 |
||
2489 |
// nothing to copy |
|
2490 |
__ mov(R0, 0); |
|
2491 |
||
2492 |
__ pop(caller_saved_regs); |
|
2493 |
__ pop(PC); |
|
2494 |
||
2495 |
// ======== begin loop ======== |
|
2496 |
// (Loop is rotated; its entry is load_element.) |
|
2497 |
__ align(OptoLoopAlignment); |
|
2498 |
__ BIND(store_element); |
|
2499 |
if (UseCompressedOops) { |
|
49950
7b916885654d
8201786: Modularize interpreter GC barriers: leftovers for ARM32
shade
parents:
49674
diff
changeset
|
2500 |
__ store_heap_oop(Address(to, BytesPerHeapOop, post_indexed), R5); // store the oop, changes flags |
42664 | 2501 |
__ subs_32(count,count,1); |
2502 |
} else { |
|
2503 |
__ subs_32(count,count,1); |
|
2504 |
__ str(R5, Address(to, BytesPerHeapOop, post_indexed)); // store the oop |
|
2505 |
} |
|
49484
ee8fa73b90f9
8198949: Modularize arraycopy stub routine GC barriers
eosterlund
parents:
49455
diff
changeset
|
2506 |
__ b(do_epilogue, eq); // count exhausted |
42664 | 2507 |
|
2508 |
// ======== loop entry is here ======== |
|
2509 |
__ BIND(load_element); |
|
2510 |
__ load_heap_oop(R5, Address(from, BytesPerHeapOop, post_indexed)); // load the oop |
|
2511 |
__ cbz(R5, store_element); // NULL |
|
2512 |
||
2513 |
__ load_klass(R6, R5); |
|
2514 |
||
2515 |
generate_type_check(R6, R3_ckoff, R4_ckval, /*tmps*/ R12, R8, R9, |
|
2516 |
// branch to this on success: |
|
2517 |
store_element); |
|
2518 |
// ======== end loop ======== |
|
2519 |
||
2520 |
// It was a real error; we must depend on the caller to finish the job. |
|
2521 |
// Register count has number of *remaining* oops, saved_count number of *total* oops. |
|
2522 |
// Emit GC store barriers for the oops we have copied |
|
2523 |
// and report their number to the caller (0 or (-1^n)) |
|
2524 |
__ BIND(fail); |
|
2525 |
||
2526 |
// Note: fail marked by the fact that count differs from saved_count |
|
2527 |
||
49484
ee8fa73b90f9
8198949: Modularize arraycopy stub routine GC barriers
eosterlund
parents:
49455
diff
changeset
|
2528 |
__ BIND(do_epilogue); |
42664 | 2529 |
|
52351 | 2530 |
Register copied = R4; // saved |
42664 | 2531 |
Label L_not_copied; |
2532 |
||
2533 |
__ subs_32(copied, saved_count, count); // copied count (in saved reg) |
|
2534 |
__ b(L_not_copied, eq); // nothing was copied, skip post barrier |
|
2535 |
__ sub(to, to, AsmOperand(copied, lsl, LogBytesPerHeapOop)); // initial to value |
|
2536 |
__ mov(R12, copied); // count arg scratched by post barrier |
|
2537 |
||
49674
c39f20946b37
8200627: aarch32 - Broken build after JDK-8198949
dpochepk
parents:
49484
diff
changeset
|
2538 |
bs->arraycopy_epilogue(_masm, decorators, true, to, R12, R3); |
42664 | 2539 |
|
2540 |
assert_different_registers(R3,R12,LR,copied,saved_count); |
|
2541 |
inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr, R3, R12); |
|
2542 |
||
2543 |
__ BIND(L_not_copied); |
|
2544 |
__ cmp_32(copied, saved_count); // values preserved in saved registers |
|
2545 |
||
2546 |
__ mov(R0, 0, eq); // 0 if all copied |
|
2547 |
__ mvn(R0, copied, ne); // else NOT(copied) |
|
2548 |
__ pop(caller_saved_regs); |
|
2549 |
__ pop(PC); |
|
2550 |
||
2551 |
return start; |
|
2552 |
} |
|
2553 |
||
2554 |
// Perform range checks on the proposed arraycopy. |
|
2555 |
// Kills the two temps, but nothing else. |
|
2556 |
void arraycopy_range_checks(Register src, // source array oop |
|
2557 |
Register src_pos, // source position (32-bit int) |
|
2558 |
Register dst, // destination array oop |
|
2559 |
Register dst_pos, // destination position (32-bit int) |
|
2560 |
Register length, // length of copy (32-bit int) |
|
2561 |
Register temp1, Register temp2, |
|
2562 |
Label& L_failed) { |
|
2563 |
||
2564 |
BLOCK_COMMENT("arraycopy_range_checks:"); |
|
2565 |
||
2566 |
// if (src_pos + length > arrayOop(src)->length() ) FAIL; |
|
2567 |
||
2568 |
const Register array_length = temp1; // scratch |
|
2569 |
const Register end_pos = temp2; // scratch |
|
2570 |
||
2571 |
__ add_32(end_pos, length, src_pos); // src_pos + length |
|
2572 |
__ ldr_s32(array_length, Address(src, arrayOopDesc::length_offset_in_bytes())); |
|
2573 |
__ cmp_32(end_pos, array_length); |
|
2574 |
__ b(L_failed, hi); |
|
2575 |
||
2576 |
// if (dst_pos + length > arrayOop(dst)->length() ) FAIL; |
|
2577 |
__ add_32(end_pos, length, dst_pos); // dst_pos + length |
|
2578 |
__ ldr_s32(array_length, Address(dst, arrayOopDesc::length_offset_in_bytes())); |
|
2579 |
__ cmp_32(end_pos, array_length); |
|
2580 |
__ b(L_failed, hi); |
|
2581 |
||
2582 |
BLOCK_COMMENT("arraycopy_range_checks done"); |
|
2583 |
} |
|
2584 |
||
2585 |
// |
|
2586 |
// Generate generic array copy stubs |
|
2587 |
// |
|
2588 |
// Input: |
|
2589 |
// R0 - src oop |
|
2590 |
// R1 - src_pos (32-bit int) |
|
2591 |
// R2 - dst oop |
|
2592 |
// R3 - dst_pos (32-bit int) |
|
52351 | 2593 |
// SP[0] - element count (32-bit int) |
42664 | 2594 |
// |
2595 |
// Output: (32-bit int) |
|
2596 |
// R0 == 0 - success |
|
2597 |
// R0 < 0 - need to call System.arraycopy |
|
2598 |
// |
|
2599 |
address generate_generic_copy(const char *name) { |
|
2600 |
Label L_failed, L_objArray; |
|
2601 |
||
2602 |
// Input registers |
|
2603 |
const Register src = R0; // source array oop |
|
2604 |
const Register src_pos = R1; // source position |
|
2605 |
const Register dst = R2; // destination array oop |
|
2606 |
const Register dst_pos = R3; // destination position |
|
2607 |
||
2608 |
// registers used as temp |
|
2609 |
const Register R5_src_klass = R5; // source array klass |
|
2610 |
const Register R6_dst_klass = R6; // destination array klass |
|
52351 | 2611 |
const Register R_lh = altFP_7_11; // layout handler |
42664 | 2612 |
const Register R8_temp = R8; |
2613 |
||
2614 |
__ align(CodeEntryAlignment); |
|
2615 |
StubCodeMark mark(this, "StubRoutines", name); |
|
2616 |
address start = __ pc(); |
|
2617 |
||
2618 |
__ zap_high_non_significant_bits(R1); |
|
2619 |
__ zap_high_non_significant_bits(R3); |
|
2620 |
__ zap_high_non_significant_bits(R4); |
|
2621 |
||
2622 |
int pushed = 0; |
|
2623 |
const RegisterSet saved_regs = RegisterSet(R4,R6) | RegisterSet(R8,R9) | altFP_7_11; |
|
2624 |
__ push(saved_regs); |
|
2625 |
assert(saved_regs.size() == 6, "check the count"); |
|
2626 |
pushed+=6; |
|
2627 |
||
2628 |
// bump this on entry, not on exit: |
|
2629 |
inc_counter_np(SharedRuntime::_generic_array_copy_ctr, R5, R12); |
|
2630 |
||
2631 |
const Register length = R4; // elements count |
|
2632 |
__ ldr(length, Address(SP,4*pushed)); |
|
2633 |
||
2634 |
||
2635 |
//----------------------------------------------------------------------- |
|
2636 |
// Assembler stubs will be used for this call to arraycopy |
|
2637 |
// if the following conditions are met: |
|
2638 |
// |
|
2639 |
// (1) src and dst must not be null. |
|
2640 |
// (2) src_pos must not be negative. |
|
2641 |
// (3) dst_pos must not be negative. |
|
2642 |
// (4) length must not be negative. |
|
2643 |
// (5) src klass and dst klass should be the same and not NULL. |
|
2644 |
// (6) src and dst should be arrays. |
|
2645 |
// (7) src_pos + length must not exceed length of src. |
|
2646 |
// (8) dst_pos + length must not exceed length of dst. |
|
2647 |
BLOCK_COMMENT("arraycopy initial argument checks"); |
|
2648 |
||
2649 |
// if (src == NULL) return -1; |
|
2650 |
__ cbz(src, L_failed); |
|
2651 |
||
2652 |
// if (src_pos < 0) return -1; |
|
2653 |
__ cmp_32(src_pos, 0); |
|
2654 |
__ b(L_failed, lt); |
|
2655 |
||
2656 |
// if (dst == NULL) return -1; |
|
2657 |
__ cbz(dst, L_failed); |
|
2658 |
||
2659 |
// if (dst_pos < 0) return -1; |
|
2660 |
__ cmp_32(dst_pos, 0); |
|
2661 |
__ b(L_failed, lt); |
|
2662 |
||
2663 |
// if (length < 0) return -1; |
|
2664 |
__ cmp_32(length, 0); |
|
2665 |
__ b(L_failed, lt); |
|
2666 |
||
2667 |
BLOCK_COMMENT("arraycopy argument klass checks"); |
|
2668 |
// get src->klass() |
|
2669 |
__ load_klass(R5_src_klass, src); |
|
2670 |
||
2671 |
// Load layout helper |
|
2672 |
// |
|
2673 |
// |array_tag| | header_size | element_type | |log2_element_size| |
|
2674 |
// 32 30 24 16 8 2 0 |
|
2675 |
// |
|
2676 |
// array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0 |
|
2677 |
// |
|
2678 |
||
2679 |
int lh_offset = in_bytes(Klass::layout_helper_offset()); |
|
2680 |
__ ldr_u32(R_lh, Address(R5_src_klass, lh_offset)); |
|
2681 |
||
2682 |
__ load_klass(R6_dst_klass, dst); |
|
2683 |
||
2684 |
// Handle objArrays completely differently... |
|
2685 |
juint objArray_lh = Klass::array_layout_helper(T_OBJECT); |
|
2686 |
__ mov_slow(R8_temp, objArray_lh); |
|
2687 |
__ cmp_32(R_lh, R8_temp); |
|
2688 |
__ b(L_objArray,eq); |
|
2689 |
||
2690 |
// if (src->klass() != dst->klass()) return -1; |
|
2691 |
__ cmp(R5_src_klass, R6_dst_klass); |
|
2692 |
__ b(L_failed, ne); |
|
2693 |
||
2694 |
// if (!src->is_Array()) return -1; |
|
2695 |
__ cmp_32(R_lh, Klass::_lh_neutral_value); // < 0 |
|
2696 |
__ b(L_failed, ge); |
|
2697 |
||
2698 |
arraycopy_range_checks(src, src_pos, dst, dst_pos, length, |
|
2699 |
R8_temp, R6_dst_klass, L_failed); |
|
2700 |
||
2701 |
{ |
|
2702 |
// TypeArrayKlass |
|
2703 |
// |
|
2704 |
// src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize); |
|
2705 |
// dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize); |
|
2706 |
// |
|
2707 |
||
2708 |
const Register R6_offset = R6_dst_klass; // array offset |
|
2709 |
const Register R12_elsize = R12; // log2 element size |
|
2710 |
||
2711 |
__ logical_shift_right(R6_offset, R_lh, Klass::_lh_header_size_shift); |
|
2712 |
__ andr(R6_offset, R6_offset, (unsigned int)Klass::_lh_header_size_mask); // array_offset |
|
2713 |
__ add(src, src, R6_offset); // src array offset |
|
2714 |
__ add(dst, dst, R6_offset); // dst array offset |
|
2715 |
__ andr(R12_elsize, R_lh, (unsigned int)Klass::_lh_log2_element_size_mask); // log2 element size |
|
2716 |
||
2717 |
// next registers should be set before the jump to corresponding stub |
|
2718 |
const Register from = R0; // source array address |
|
2719 |
const Register to = R1; // destination array address |
|
2720 |
const Register count = R2; // elements count |
|
2721 |
||
2722 |
// 'from', 'to', 'count' registers should be set in this order |
|
2723 |
// since they are the same as 'src', 'src_pos', 'dst'. |
|
2724 |
||
2725 |
||
2726 |
BLOCK_COMMENT("scale indexes to element size"); |
|
2727 |
__ add(from, src, AsmOperand(src_pos, lsl, R12_elsize)); // src_addr |
|
2728 |
__ add(to, dst, AsmOperand(dst_pos, lsl, R12_elsize)); // dst_addr |
|
2729 |
||
2730 |
__ mov(count, length); // length |
|
2731 |
||
2732 |
// XXX optim: avoid later push in arraycopy variants ? |
|
2733 |
||
2734 |
__ pop(saved_regs); |
|
2735 |
||
2736 |
BLOCK_COMMENT("choose copy loop based on element size"); |
|
2737 |
__ cmp(R12_elsize, 0); |
|
2738 |
__ b(StubRoutines::_jbyte_arraycopy,eq); |
|
2739 |
||
2740 |
__ cmp(R12_elsize, LogBytesPerShort); |
|
2741 |
__ b(StubRoutines::_jshort_arraycopy,eq); |
|
2742 |
||
2743 |
__ cmp(R12_elsize, LogBytesPerInt); |
|
2744 |
__ b(StubRoutines::_jint_arraycopy,eq); |
|
2745 |
||
2746 |
__ b(StubRoutines::_jlong_arraycopy); |
|
2747 |
||
2748 |
} |
|
2749 |
||
2750 |
// ObjArrayKlass |
|
2751 |
__ BIND(L_objArray); |
|
2752 |
// live at this point: R5_src_klass, R6_dst_klass, src[_pos], dst[_pos], length |
|
2753 |
||
2754 |
Label L_plain_copy, L_checkcast_copy; |
|
2755 |
// test array classes for subtyping |
|
2756 |
__ cmp(R5_src_klass, R6_dst_klass); // usual case is exact equality |
|
2757 |
__ b(L_checkcast_copy, ne); |
|
2758 |
||
2759 |
BLOCK_COMMENT("Identically typed arrays"); |
|
2760 |
{ |
|
2761 |
// Identically typed arrays can be copied without element-wise checks. |
|
2762 |
arraycopy_range_checks(src, src_pos, dst, dst_pos, length, |
|
2763 |
R8_temp, R_lh, L_failed); |
|
2764 |
||
2765 |
// next registers should be set before the jump to corresponding stub |
|
2766 |
const Register from = R0; // source array address |
|
2767 |
const Register to = R1; // destination array address |
|
2768 |
const Register count = R2; // elements count |
|
2769 |
||
2770 |
__ add(src, src, arrayOopDesc::base_offset_in_bytes(T_OBJECT)); //src offset |
|
2771 |
__ add(dst, dst, arrayOopDesc::base_offset_in_bytes(T_OBJECT)); //dst offset |
|
2772 |
__ add_ptr_scaled_int32(from, src, src_pos, LogBytesPerHeapOop); // src_addr |
|
2773 |
__ add_ptr_scaled_int32(to, dst, dst_pos, LogBytesPerHeapOop); // dst_addr |
|
2774 |
__ BIND(L_plain_copy); |
|
2775 |
__ mov(count, length); |
|
2776 |
||
2777 |
__ pop(saved_regs); // XXX optim: avoid later push in oop_arraycopy ? |
|
2778 |
__ b(StubRoutines::_oop_arraycopy); |
|
2779 |
} |
|
2780 |
||
2781 |
{ |
|
2782 |
__ BIND(L_checkcast_copy); |
|
2783 |
// live at this point: R5_src_klass, R6_dst_klass |
|
2784 |
||
2785 |
// Before looking at dst.length, make sure dst is also an objArray. |
|
2786 |
__ ldr_u32(R8_temp, Address(R6_dst_klass, lh_offset)); |
|
2787 |
__ cmp_32(R_lh, R8_temp); |
|
2788 |
__ b(L_failed, ne); |
|
2789 |
||
2790 |
// It is safe to examine both src.length and dst.length. |
|
2791 |
||
2792 |
arraycopy_range_checks(src, src_pos, dst, dst_pos, length, |
|
2793 |
R8_temp, R_lh, L_failed); |
|
2794 |
||
2795 |
// next registers should be set before the jump to corresponding stub |
|
2796 |
const Register from = R0; // source array address |
|
2797 |
const Register to = R1; // destination array address |
|
2798 |
const Register count = R2; // elements count |
|
2799 |
||
2800 |
// Marshal the base address arguments now, freeing registers. |
|
2801 |
__ add(src, src, arrayOopDesc::base_offset_in_bytes(T_OBJECT)); //src offset |
|
2802 |
__ add(dst, dst, arrayOopDesc::base_offset_in_bytes(T_OBJECT)); //dst offset |
|
2803 |
__ add_ptr_scaled_int32(from, src, src_pos, LogBytesPerHeapOop); // src_addr |
|
2804 |
__ add_ptr_scaled_int32(to, dst, dst_pos, LogBytesPerHeapOop); // dst_addr |
|
2805 |
||
2806 |
__ mov(count, length); // length (reloaded) |
|
2807 |
||
2808 |
Register sco_temp = R3; // this register is free now |
|
2809 |
assert_different_registers(from, to, count, sco_temp, |
|
2810 |
R6_dst_klass, R5_src_klass); |
|
2811 |
||
2812 |
// Generate the type check. |
|
2813 |
int sco_offset = in_bytes(Klass::super_check_offset_offset()); |
|
2814 |
__ ldr_u32(sco_temp, Address(R6_dst_klass, sco_offset)); |
|
2815 |
generate_type_check(R5_src_klass, sco_temp, R6_dst_klass, |
|
2816 |
R8_temp, R9, |
|
52351 | 2817 |
R12, |
42664 | 2818 |
L_plain_copy); |
2819 |
||
2820 |
// Fetch destination element klass from the ObjArrayKlass header. |
|
2821 |
int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset()); |
|
2822 |
||
2823 |
// the checkcast_copy loop needs two extra arguments: |
|
52351 | 2824 |
const Register Rdst_elem_klass = R3; |
42664 | 2825 |
__ ldr(Rdst_elem_klass, Address(R6_dst_klass, ek_offset)); // dest elem klass |
2826 |
__ pop(saved_regs); // XXX optim: avoid later push in oop_arraycopy ? |
|
2827 |
__ str(Rdst_elem_klass, Address(SP,0)); // dest elem klass argument |
|
2828 |
__ ldr_u32(R3, Address(Rdst_elem_klass, sco_offset)); // sco of elem klass |
|
2829 |
__ b(StubRoutines::_checkcast_arraycopy); |
|
2830 |
} |
|
2831 |
||
2832 |
__ BIND(L_failed); |
|
2833 |
||
2834 |
__ pop(saved_regs); |
|
2835 |
__ mvn(R0, 0); // failure, with 0 copied |
|
2836 |
__ ret(); |
|
2837 |
||
2838 |
return start; |
|
2839 |
} |
|
2840 |
||
2841 |
// Safefetch stubs. |
|
2842 |
void generate_safefetch(const char* name, int size, address* entry, address* fault_pc, address* continuation_pc) { |
|
2843 |
// safefetch signatures: |
|
2844 |
// int SafeFetch32(int* adr, int errValue); |
|
2845 |
// intptr_t SafeFetchN (intptr_t* adr, intptr_t errValue); |
|
2846 |
// |
|
2847 |
// arguments: |
|
2848 |
// R0 = adr |
|
2849 |
// R1 = errValue |
|
2850 |
// |
|
2851 |
// result: |
|
2852 |
// R0 = *adr or errValue |
|
2853 |
||
2854 |
StubCodeMark mark(this, "StubRoutines", name); |
|
2855 |
||
2856 |
// Entry point, pc or function descriptor. |
|
2857 |
*entry = __ pc(); |
|
2858 |
||
2859 |
// Load *adr into c_rarg2, may fault. |
|
2860 |
*fault_pc = __ pc(); |
|
2861 |
||
2862 |
switch (size) { |
|
2863 |
case 4: // int32_t |
|
2864 |
__ ldr_s32(R1, Address(R0)); |
|
2865 |
break; |
|
2866 |
||
2867 |
case 8: // int64_t |
|
2868 |
Unimplemented(); |
|
2869 |
break; |
|
2870 |
||
2871 |
default: |
|
2872 |
ShouldNotReachHere(); |
|
2873 |
} |
|
2874 |
||
2875 |
// return errValue or *adr |
|
2876 |
*continuation_pc = __ pc(); |
|
2877 |
__ mov(R0, R1); |
|
2878 |
__ ret(); |
|
2879 |
} |
|
2880 |
||
2881 |
void generate_arraycopy_stubs() { |
|
2882 |
||
2883 |
// Note: the disjoint stubs must be generated first, some of |
|
2884 |
// the conjoint stubs use them. |
|
2885 |
||
2886 |
bool status = false; // non failing C2 stubs need not return a status in R0 |
|
2887 |
||
2888 |
#ifdef TEST_C2_GENERIC_ARRAYCOPY /* Internal development flag */ |
|
2889 |
// With this flag, the C2 stubs are tested by generating calls to |
|
2890 |
// generic_arraycopy instead of Runtime1::arraycopy |
|
2891 |
||
2892 |
// Runtime1::arraycopy return a status in R0 (0 if OK, else ~copied) |
|
2893 |
// and the result is tested to see whether the arraycopy stub should |
|
2894 |
// be called. |
|
2895 |
||
2896 |
// When we test arraycopy this way, we must generate extra code in the |
|
2897 |
// arraycopy methods callable from C2 generic_arraycopy to set the |
|
2898 |
// status to 0 for those who always succeed (calling the slow path stub might |
|
2899 |
// lead to errors since the copy has already been performed). |
|
2900 |
||
2901 |
status = true; // generate a status compatible with C1 calls |
|
2902 |
#endif |
|
2903 |
||
55490
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
2904 |
address ucm_common_error_exit = generate_unsafecopy_common_error_exit(); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
2905 |
UnsafeCopyMemory::set_common_exit_stub_pc(ucm_common_error_exit); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
2906 |
|
42664 | 2907 |
// these need always status in case they are called from generic_arraycopy |
2908 |
StubRoutines::_jbyte_disjoint_arraycopy = generate_primitive_copy(false, "jbyte_disjoint_arraycopy", true, 1, true); |
|
2909 |
StubRoutines::_jshort_disjoint_arraycopy = generate_primitive_copy(false, "jshort_disjoint_arraycopy", true, 2, true); |
|
2910 |
StubRoutines::_jint_disjoint_arraycopy = generate_primitive_copy(false, "jint_disjoint_arraycopy", true, 4, true); |
|
2911 |
StubRoutines::_jlong_disjoint_arraycopy = generate_primitive_copy(false, "jlong_disjoint_arraycopy", true, 8, true); |
|
2912 |
StubRoutines::_oop_disjoint_arraycopy = generate_oop_copy (false, "oop_disjoint_arraycopy", true, true); |
|
2913 |
||
2914 |
StubRoutines::_arrayof_jbyte_disjoint_arraycopy = generate_primitive_copy(true, "arrayof_jbyte_disjoint_arraycopy", status, 1, true); |
|
2915 |
StubRoutines::_arrayof_jshort_disjoint_arraycopy = generate_primitive_copy(true, "arrayof_jshort_disjoint_arraycopy",status, 2, true); |
|
2916 |
StubRoutines::_arrayof_jint_disjoint_arraycopy = generate_primitive_copy(true, "arrayof_jint_disjoint_arraycopy", status, 4, true); |
|
2917 |
StubRoutines::_arrayof_jlong_disjoint_arraycopy = generate_primitive_copy(true, "arrayof_jlong_disjoint_arraycopy", status, 8, true); |
|
2918 |
StubRoutines::_arrayof_oop_disjoint_arraycopy = generate_oop_copy (true, "arrayof_oop_disjoint_arraycopy", status, true); |
|
2919 |
||
2920 |
// these need always status in case they are called from generic_arraycopy |
|
2921 |
StubRoutines::_jbyte_arraycopy = generate_primitive_copy(false, "jbyte_arraycopy", true, 1, false, StubRoutines::_jbyte_disjoint_arraycopy); |
|
2922 |
StubRoutines::_jshort_arraycopy = generate_primitive_copy(false, "jshort_arraycopy", true, 2, false, StubRoutines::_jshort_disjoint_arraycopy); |
|
2923 |
StubRoutines::_jint_arraycopy = generate_primitive_copy(false, "jint_arraycopy", true, 4, false, StubRoutines::_jint_disjoint_arraycopy); |
|
2924 |
StubRoutines::_jlong_arraycopy = generate_primitive_copy(false, "jlong_arraycopy", true, 8, false, StubRoutines::_jlong_disjoint_arraycopy); |
|
2925 |
StubRoutines::_oop_arraycopy = generate_oop_copy (false, "oop_arraycopy", true, false, StubRoutines::_oop_disjoint_arraycopy); |
|
2926 |
||
2927 |
StubRoutines::_arrayof_jbyte_arraycopy = generate_primitive_copy(true, "arrayof_jbyte_arraycopy", status, 1, false, StubRoutines::_arrayof_jbyte_disjoint_arraycopy); |
|
2928 |
StubRoutines::_arrayof_jshort_arraycopy = generate_primitive_copy(true, "arrayof_jshort_arraycopy", status, 2, false, StubRoutines::_arrayof_jshort_disjoint_arraycopy); |
|
2929 |
#ifdef _LP64 |
|
2930 |
// since sizeof(jint) < sizeof(HeapWord), there's a different flavor: |
|
2931 |
StubRoutines::_arrayof_jint_arraycopy = generate_primitive_copy(true, "arrayof_jint_arraycopy", status, 4, false, StubRoutines::_arrayof_jint_disjoint_arraycopy); |
|
2932 |
#else |
|
2933 |
StubRoutines::_arrayof_jint_arraycopy = StubRoutines::_jint_arraycopy; |
|
2934 |
#endif |
|
2935 |
if (BytesPerHeapOop < HeapWordSize) { |
|
2936 |
StubRoutines::_arrayof_oop_arraycopy = generate_oop_copy (true, "arrayof_oop_arraycopy", status, false, StubRoutines::_arrayof_oop_disjoint_arraycopy); |
|
2937 |
} else { |
|
2938 |
StubRoutines::_arrayof_oop_arraycopy = StubRoutines::_oop_arraycopy; |
|
2939 |
} |
|
2940 |
StubRoutines::_arrayof_jlong_arraycopy = StubRoutines::_jlong_arraycopy; |
|
2941 |
||
2942 |
StubRoutines::_checkcast_arraycopy = generate_checkcast_copy("checkcast_arraycopy"); |
|
2943 |
StubRoutines::_unsafe_arraycopy = generate_unsafe_copy("unsafe_arraycopy"); |
|
2944 |
StubRoutines::_generic_arraycopy = generate_generic_copy("generic_arraycopy"); |
|
2945 |
||
2946 |
||
2947 |
} |
|
2948 |
||
2949 |
#define COMPILE_CRYPTO |
|
2950 |
#include "stubRoutinesCrypto_arm.cpp" |
|
2951 |
||
2952 |
private: |
|
2953 |
||
2954 |
#undef __ |
|
2955 |
#define __ masm-> |
|
2956 |
||
2957 |
//------------------------------------------------------------------------------------------------------------------------ |
|
2958 |
// Continuation point for throwing of implicit exceptions that are not handled in |
|
2959 |
// the current activation. Fabricates an exception oop and initiates normal |
|
2960 |
// exception dispatching in this frame. |
|
2961 |
address generate_throw_exception(const char* name, address runtime_entry) { |
|
2962 |
int insts_size = 128; |
|
2963 |
int locs_size = 32; |
|
2964 |
CodeBuffer code(name, insts_size, locs_size); |
|
2965 |
OopMapSet* oop_maps; |
|
2966 |
int frame_size; |
|
2967 |
int frame_complete; |
|
2968 |
||
2969 |
oop_maps = new OopMapSet(); |
|
2970 |
MacroAssembler* masm = new MacroAssembler(&code); |
|
2971 |
||
2972 |
address start = __ pc(); |
|
2973 |
||
2974 |
frame_size = 2; |
|
2975 |
__ mov(Rexception_pc, LR); |
|
2976 |
__ raw_push(FP, LR); |
|
2977 |
||
2978 |
frame_complete = __ pc() - start; |
|
2979 |
||
2980 |
// Any extra arguments are already supposed to be R1 and R2 |
|
2981 |
__ mov(R0, Rthread); |
|
2982 |
||
2983 |
int pc_offset = __ set_last_Java_frame(SP, FP, false, Rtemp); |
|
2984 |
assert(((__ pc()) - start) == __ offset(), "warning: start differs from code_begin"); |
|
2985 |
__ call(runtime_entry); |
|
2986 |
if (pc_offset == -1) { |
|
2987 |
pc_offset = __ offset(); |
|
2988 |
} |
|
2989 |
||
2990 |
// Generate oop map |
|
2991 |
OopMap* map = new OopMap(frame_size*VMRegImpl::slots_per_word, 0); |
|
2992 |
oop_maps->add_gc_map(pc_offset, map); |
|
2993 |
__ reset_last_Java_frame(Rtemp); // Rtemp free since scratched by far call |
|
2994 |
||
2995 |
__ raw_pop(FP, LR); |
|
2996 |
__ jump(StubRoutines::forward_exception_entry(), relocInfo::runtime_call_type, Rtemp); |
|
2997 |
||
2998 |
RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &code, frame_complete, |
|
2999 |
frame_size, oop_maps, false); |
|
3000 |
return stub->entry_point(); |
|
3001 |
} |
|
3002 |
||
3003 |
//--------------------------------------------------------------------------- |
|
3004 |
// Initialization |
|
3005 |
||
3006 |
void generate_initial() { |
|
3007 |
// Generates all stubs and initializes the entry points |
|
3008 |
||
3009 |
//------------------------------------------------------------------------------------------------------------------------ |
|
3010 |
// entry points that exist in all platforms |
|
3011 |
// Note: This is code that could be shared among different platforms - however the benefit seems to be smaller than |
|
3012 |
// the disadvantage of having a much more complicated generator structure. See also comment in stubRoutines.hpp. |
|
3013 |
StubRoutines::_forward_exception_entry = generate_forward_exception(); |
|
3014 |
||
3015 |
StubRoutines::_call_stub_entry = |
|
3016 |
generate_call_stub(StubRoutines::_call_stub_return_address); |
|
3017 |
// is referenced by megamorphic call |
|
3018 |
StubRoutines::_catch_exception_entry = generate_catch_exception(); |
|
3019 |
||
3020 |
// stub for throwing stack overflow error used both by interpreter and compiler |
|
3021 |
StubRoutines::_throw_StackOverflowError_entry = generate_throw_exception("StackOverflowError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_StackOverflowError)); |
|
3022 |
||
3023 |
// integer division used both by interpreter and compiler |
|
3024 |
StubRoutines::Arm::_idiv_irem_entry = generate_idiv_irem(); |
|
3025 |
||
3026 |
StubRoutines::_atomic_add_entry = generate_atomic_add(); |
|
3027 |
StubRoutines::_atomic_xchg_entry = generate_atomic_xchg(); |
|
3028 |
StubRoutines::_atomic_cmpxchg_entry = generate_atomic_cmpxchg(); |
|
3029 |
StubRoutines::_atomic_cmpxchg_long_entry = generate_atomic_cmpxchg_long(); |
|
3030 |
StubRoutines::_atomic_load_long_entry = generate_atomic_load_long(); |
|
3031 |
StubRoutines::_atomic_store_long_entry = generate_atomic_store_long(); |
|
3032 |
} |
|
3033 |
||
3034 |
void generate_all() { |
|
3035 |
// Generates all stubs and initializes the entry points |
|
3036 |
||
3037 |
#ifdef COMPILER2 |
|
3038 |
// Generate partial_subtype_check first here since its code depends on |
|
3039 |
// UseZeroBaseCompressedOops which is defined after heap initialization. |
|
3040 |
StubRoutines::Arm::_partial_subtype_check = generate_partial_subtype_check(); |
|
3041 |
#endif |
|
3042 |
// These entry points require SharedInfo::stack0 to be set up in non-core builds |
|
3043 |
// and need to be relocatable, so they each fabricate a RuntimeStub internally. |
|
3044 |
StubRoutines::_throw_AbstractMethodError_entry = generate_throw_exception("AbstractMethodError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_AbstractMethodError)); |
|
3045 |
StubRoutines::_throw_IncompatibleClassChangeError_entry= generate_throw_exception("IncompatibleClassChangeError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_IncompatibleClassChangeError)); |
|
3046 |
StubRoutines::_throw_NullPointerException_at_call_entry= generate_throw_exception("NullPointerException at call throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException_at_call)); |
|
3047 |
||
3048 |
//------------------------------------------------------------------------------------------------------------------------ |
|
3049 |
// entry points that are platform specific |
|
3050 |
||
3051 |
// support for verify_oop (must happen after universe_init) |
|
3052 |
StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop(); |
|
3053 |
||
3054 |
// arraycopy stubs used by compilers |
|
3055 |
generate_arraycopy_stubs(); |
|
3056 |
||
3057 |
// Safefetch stubs. |
|
3058 |
generate_safefetch("SafeFetch32", sizeof(int), &StubRoutines::_safefetch32_entry, |
|
3059 |
&StubRoutines::_safefetch32_fault_pc, |
|
3060 |
&StubRoutines::_safefetch32_continuation_pc); |
|
3061 |
assert (sizeof(int) == wordSize, "32-bit architecture"); |
|
3062 |
StubRoutines::_safefetchN_entry = StubRoutines::_safefetch32_entry; |
|
3063 |
StubRoutines::_safefetchN_fault_pc = StubRoutines::_safefetch32_fault_pc; |
|
3064 |
StubRoutines::_safefetchN_continuation_pc = StubRoutines::_safefetch32_continuation_pc; |
|
3065 |
||
3066 |
#ifdef COMPILE_CRYPTO |
|
3067 |
// generate AES intrinsics code |
|
3068 |
if (UseAESIntrinsics) { |
|
3069 |
aes_init(); |
|
3070 |
StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock(); |
|
3071 |
StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock(); |
|
3072 |
StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt(); |
|
3073 |
StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt(); |
|
3074 |
} |
|
3075 |
#endif // COMPILE_CRYPTO |
|
3076 |
} |
|
3077 |
||
3078 |
||
3079 |
public: |
|
3080 |
StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) { |
|
3081 |
if (all) { |
|
3082 |
generate_all(); |
|
3083 |
} else { |
|
3084 |
generate_initial(); |
|
3085 |
} |
|
3086 |
} |
|
3087 |
}; // end class declaration |
|
3088 |
||
55490
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
3089 |
#define UCM_TABLE_MAX_ENTRIES 32 |
42664 | 3090 |
void StubGenerator_generate(CodeBuffer* code, bool all) { |
55490
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
3091 |
if (UnsafeCopyMemory::_table == NULL) { |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
3092 |
UnsafeCopyMemory::create_table(UCM_TABLE_MAX_ENTRIES); |
3f3dc00a69a5
8191278: MappedByteBuffer bulk access memory failures are not handled gracefully
jcm
parents:
54786
diff
changeset
|
3093 |
} |
42664 | 3094 |
StubGenerator g(code, all); |
3095 |
} |