author | vlivanov |
Fri, 29 Apr 2016 02:13:40 +0300 | |
changeset 38144 | 0976c0c5c5d3 |
parent 38138 | 8514e24123c8 |
parent 38074 | 8475fdc6dcc3 |
child 38241 | 32eab2eb41fd |
permissions | -rw-r--r-- |
14626 | 1 |
/* |
35847
990c61a50f24
8143608: Don't 64-bit align start of InstanceKlass vtable, itable, and nonstatic_oopmap on 32-bit systems
cjplummer
parents:
35232
diff
changeset
|
2 |
* Copyright (c) 1997, 2016, Oracle and/or its affiliates. All rights reserved. |
14626 | 3 |
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
4 |
* |
|
5 |
* This code is free software; you can redistribute it and/or modify it |
|
6 |
* under the terms of the GNU General Public License version 2 only, as |
|
7 |
* published by the Free Software Foundation. |
|
8 |
* |
|
9 |
* This code is distributed in the hope that it will be useful, but WITHOUT |
|
10 |
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
|
11 |
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
|
12 |
* version 2 for more details (a copy is included in the LICENSE file that |
|
13 |
* accompanied this code). |
|
14 |
* |
|
15 |
* You should have received a copy of the GNU General Public License version |
|
16 |
* 2 along with this work; if not, write to the Free Software Foundation, |
|
17 |
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
|
18 |
* |
|
19 |
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
|
20 |
* or visit www.oracle.com if you need additional information or have any |
|
21 |
* questions. |
|
22 |
* |
|
23 |
*/ |
|
24 |
||
25 |
#include "precompiled.hpp" |
|
26 |
#include "asm/assembler.hpp" |
|
27 |
#include "asm/assembler.inline.hpp" |
|
28 |
#include "compiler/disassembler.hpp" |
|
30764 | 29 |
#include "gc/shared/cardTableModRefBS.hpp" |
30 |
#include "gc/shared/collectedHeap.inline.hpp" |
|
14626 | 31 |
#include "interpreter/interpreter.hpp" |
32 |
#include "memory/resourceArea.hpp" |
|
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
33 |
#include "memory/universe.hpp" |
31849
92ca49fa9fa7
8131344: Missing klass.inline.hpp include in compiler files
tschatzl
parents:
31782
diff
changeset
|
34 |
#include "oops/klass.inline.hpp" |
14626 | 35 |
#include "prims/methodHandles.hpp" |
36 |
#include "runtime/biasedLocking.hpp" |
|
37 |
#include "runtime/interfaceSupport.hpp" |
|
38 |
#include "runtime/objectMonitor.hpp" |
|
39 |
#include "runtime/os.hpp" |
|
40 |
#include "runtime/sharedRuntime.hpp" |
|
41 |
#include "runtime/stubRoutines.hpp" |
|
34633
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
42 |
#include "runtime/thread.hpp" |
15482
470d0b0c09f1
8005915: Unify SERIALGC and INCLUDE_ALTERNATE_GCS
jprovino
parents:
15117
diff
changeset
|
43 |
#include "utilities/macros.hpp" |
470d0b0c09f1
8005915: Unify SERIALGC and INCLUDE_ALTERNATE_GCS
jprovino
parents:
15117
diff
changeset
|
44 |
#if INCLUDE_ALL_GCS |
30764 | 45 |
#include "gc/g1/g1CollectedHeap.inline.hpp" |
46 |
#include "gc/g1/g1SATBCardTableModRefBS.hpp" |
|
47 |
#include "gc/g1/heapRegion.hpp" |
|
15482
470d0b0c09f1
8005915: Unify SERIALGC and INCLUDE_ALTERNATE_GCS
jprovino
parents:
15117
diff
changeset
|
48 |
#endif // INCLUDE_ALL_GCS |
33066 | 49 |
#include "crc32c.h" |
33628 | 50 |
#ifdef COMPILER2 |
51 |
#include "opto/intrinsicnode.hpp" |
|
52 |
#endif |
|
14626 | 53 |
|
54 |
#ifdef PRODUCT |
|
55 |
#define BLOCK_COMMENT(str) /* nothing */ |
|
56 |
#define STOP(error) stop(error) |
|
57 |
#else |
|
58 |
#define BLOCK_COMMENT(str) block_comment(str) |
|
59 |
#define STOP(error) block_comment(error); stop(error) |
|
60 |
#endif |
|
61 |
||
62 |
#define BIND(label) bind(label); BLOCK_COMMENT(#label ":") |
|
63 |
||
14631
526804361522
8003250: SPARC: move MacroAssembler into separate file
twisti
parents:
14626
diff
changeset
|
64 |
#ifdef ASSERT |
526804361522
8003250: SPARC: move MacroAssembler into separate file
twisti
parents:
14626
diff
changeset
|
65 |
bool AbstractAssembler::pd_check_instruction_mark() { return true; } |
526804361522
8003250: SPARC: move MacroAssembler into separate file
twisti
parents:
14626
diff
changeset
|
66 |
#endif |
526804361522
8003250: SPARC: move MacroAssembler into separate file
twisti
parents:
14626
diff
changeset
|
67 |
|
14626 | 68 |
static Assembler::Condition reverse[] = { |
69 |
Assembler::noOverflow /* overflow = 0x0 */ , |
|
70 |
Assembler::overflow /* noOverflow = 0x1 */ , |
|
71 |
Assembler::aboveEqual /* carrySet = 0x2, below = 0x2 */ , |
|
72 |
Assembler::below /* aboveEqual = 0x3, carryClear = 0x3 */ , |
|
73 |
Assembler::notZero /* zero = 0x4, equal = 0x4 */ , |
|
74 |
Assembler::zero /* notZero = 0x5, notEqual = 0x5 */ , |
|
75 |
Assembler::above /* belowEqual = 0x6 */ , |
|
76 |
Assembler::belowEqual /* above = 0x7 */ , |
|
77 |
Assembler::positive /* negative = 0x8 */ , |
|
78 |
Assembler::negative /* positive = 0x9 */ , |
|
79 |
Assembler::noParity /* parity = 0xa */ , |
|
80 |
Assembler::parity /* noParity = 0xb */ , |
|
81 |
Assembler::greaterEqual /* less = 0xc */ , |
|
82 |
Assembler::less /* greaterEqual = 0xd */ , |
|
83 |
Assembler::greater /* lessEqual = 0xe */ , |
|
84 |
Assembler::lessEqual /* greater = 0xf, */ |
|
85 |
||
86 |
}; |
|
87 |
||
88 |
||
89 |
// Implementation of MacroAssembler |
|
90 |
||
91 |
// First all the versions that have distinct versions depending on 32/64 bit |
|
92 |
// Unless the difference is trivial (1 line or so). |
|
93 |
||
94 |
#ifndef _LP64 |
|
95 |
||
96 |
// 32bit versions |
|
97 |
||
98 |
Address MacroAssembler::as_Address(AddressLiteral adr) { |
|
99 |
return Address(adr.target(), adr.rspec()); |
|
100 |
} |
|
101 |
||
102 |
Address MacroAssembler::as_Address(ArrayAddress adr) { |
|
103 |
return Address::make_array(adr); |
|
104 |
} |
|
105 |
||
106 |
void MacroAssembler::call_VM_leaf_base(address entry_point, |
|
107 |
int number_of_arguments) { |
|
108 |
call(RuntimeAddress(entry_point)); |
|
109 |
increment(rsp, number_of_arguments * wordSize); |
|
110 |
} |
|
111 |
||
112 |
void MacroAssembler::cmpklass(Address src1, Metadata* obj) { |
|
113 |
cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate()); |
|
114 |
} |
|
115 |
||
116 |
void MacroAssembler::cmpklass(Register src1, Metadata* obj) { |
|
117 |
cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate()); |
|
118 |
} |
|
119 |
||
120 |
void MacroAssembler::cmpoop(Address src1, jobject obj) { |
|
121 |
cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate()); |
|
122 |
} |
|
123 |
||
124 |
void MacroAssembler::cmpoop(Register src1, jobject obj) { |
|
125 |
cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate()); |
|
126 |
} |
|
127 |
||
128 |
void MacroAssembler::extend_sign(Register hi, Register lo) { |
|
129 |
// According to Intel Doc. AP-526, "Integer Divide", p.18. |
|
130 |
if (VM_Version::is_P6() && hi == rdx && lo == rax) { |
|
131 |
cdql(); |
|
132 |
} else { |
|
133 |
movl(hi, lo); |
|
134 |
sarl(hi, 31); |
|
135 |
} |
|
136 |
} |
|
137 |
||
138 |
void MacroAssembler::jC2(Register tmp, Label& L) { |
|
139 |
// set parity bit if FPU flag C2 is set (via rax) |
|
140 |
save_rax(tmp); |
|
141 |
fwait(); fnstsw_ax(); |
|
142 |
sahf(); |
|
143 |
restore_rax(tmp); |
|
144 |
// branch |
|
145 |
jcc(Assembler::parity, L); |
|
146 |
} |
|
147 |
||
148 |
void MacroAssembler::jnC2(Register tmp, Label& L) { |
|
149 |
// set parity bit if FPU flag C2 is set (via rax) |
|
150 |
save_rax(tmp); |
|
151 |
fwait(); fnstsw_ax(); |
|
152 |
sahf(); |
|
153 |
restore_rax(tmp); |
|
154 |
// branch |
|
155 |
jcc(Assembler::noParity, L); |
|
156 |
} |
|
157 |
||
158 |
// 32bit can do a case table jump in one instruction but we no longer allow the base |
|
159 |
// to be installed in the Address class |
|
160 |
void MacroAssembler::jump(ArrayAddress entry) { |
|
161 |
jmp(as_Address(entry)); |
|
162 |
} |
|
163 |
||
164 |
// Note: y_lo will be destroyed |
|
165 |
void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) { |
|
166 |
// Long compare for Java (semantics as described in JVM spec.) |
|
167 |
Label high, low, done; |
|
168 |
||
169 |
cmpl(x_hi, y_hi); |
|
170 |
jcc(Assembler::less, low); |
|
171 |
jcc(Assembler::greater, high); |
|
172 |
// x_hi is the return register |
|
173 |
xorl(x_hi, x_hi); |
|
174 |
cmpl(x_lo, y_lo); |
|
175 |
jcc(Assembler::below, low); |
|
176 |
jcc(Assembler::equal, done); |
|
177 |
||
178 |
bind(high); |
|
179 |
xorl(x_hi, x_hi); |
|
180 |
increment(x_hi); |
|
181 |
jmp(done); |
|
182 |
||
183 |
bind(low); |
|
184 |
xorl(x_hi, x_hi); |
|
185 |
decrementl(x_hi); |
|
186 |
||
187 |
bind(done); |
|
188 |
} |
|
189 |
||
190 |
void MacroAssembler::lea(Register dst, AddressLiteral src) { |
|
191 |
mov_literal32(dst, (int32_t)src.target(), src.rspec()); |
|
192 |
} |
|
193 |
||
194 |
void MacroAssembler::lea(Address dst, AddressLiteral adr) { |
|
195 |
// leal(dst, as_Address(adr)); |
|
196 |
// see note in movl as to why we must use a move |
|
197 |
mov_literal32(dst, (int32_t) adr.target(), adr.rspec()); |
|
198 |
} |
|
199 |
||
200 |
void MacroAssembler::leave() { |
|
201 |
mov(rsp, rbp); |
|
202 |
pop(rbp); |
|
203 |
} |
|
204 |
||
205 |
void MacroAssembler::lmul(int x_rsp_offset, int y_rsp_offset) { |
|
206 |
// Multiplication of two Java long values stored on the stack |
|
207 |
// as illustrated below. Result is in rdx:rax. |
|
208 |
// |
|
209 |
// rsp ---> [ ?? ] \ \ |
|
210 |
// .... | y_rsp_offset | |
|
211 |
// [ y_lo ] / (in bytes) | x_rsp_offset |
|
212 |
// [ y_hi ] | (in bytes) |
|
213 |
// .... | |
|
214 |
// [ x_lo ] / |
|
215 |
// [ x_hi ] |
|
216 |
// .... |
|
217 |
// |
|
218 |
// Basic idea: lo(result) = lo(x_lo * y_lo) |
|
219 |
// hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi) |
|
220 |
Address x_hi(rsp, x_rsp_offset + wordSize); Address x_lo(rsp, x_rsp_offset); |
|
221 |
Address y_hi(rsp, y_rsp_offset + wordSize); Address y_lo(rsp, y_rsp_offset); |
|
222 |
Label quick; |
|
223 |
// load x_hi, y_hi and check if quick |
|
224 |
// multiplication is possible |
|
225 |
movl(rbx, x_hi); |
|
226 |
movl(rcx, y_hi); |
|
227 |
movl(rax, rbx); |
|
228 |
orl(rbx, rcx); // rbx, = 0 <=> x_hi = 0 and y_hi = 0 |
|
229 |
jcc(Assembler::zero, quick); // if rbx, = 0 do quick multiply |
|
230 |
// do full multiplication |
|
231 |
// 1st step |
|
232 |
mull(y_lo); // x_hi * y_lo |
|
233 |
movl(rbx, rax); // save lo(x_hi * y_lo) in rbx, |
|
234 |
// 2nd step |
|
235 |
movl(rax, x_lo); |
|
236 |
mull(rcx); // x_lo * y_hi |
|
237 |
addl(rbx, rax); // add lo(x_lo * y_hi) to rbx, |
|
238 |
// 3rd step |
|
239 |
bind(quick); // note: rbx, = 0 if quick multiply! |
|
240 |
movl(rax, x_lo); |
|
241 |
mull(y_lo); // x_lo * y_lo |
|
242 |
addl(rdx, rbx); // correct hi(x_lo * y_lo) |
|
243 |
} |
|
244 |
||
245 |
void MacroAssembler::lneg(Register hi, Register lo) { |
|
246 |
negl(lo); |
|
247 |
adcl(hi, 0); |
|
248 |
negl(hi); |
|
249 |
} |
|
250 |
||
251 |
void MacroAssembler::lshl(Register hi, Register lo) { |
|
252 |
// Java shift left long support (semantics as described in JVM spec., p.305) |
|
253 |
// (basic idea for shift counts s >= n: x << s == (x << n) << (s - n)) |
|
254 |
// shift value is in rcx ! |
|
255 |
assert(hi != rcx, "must not use rcx"); |
|
256 |
assert(lo != rcx, "must not use rcx"); |
|
257 |
const Register s = rcx; // shift count |
|
258 |
const int n = BitsPerWord; |
|
259 |
Label L; |
|
260 |
andl(s, 0x3f); // s := s & 0x3f (s < 0x40) |
|
261 |
cmpl(s, n); // if (s < n) |
|
262 |
jcc(Assembler::less, L); // else (s >= n) |
|
263 |
movl(hi, lo); // x := x << n |
|
264 |
xorl(lo, lo); |
|
265 |
// Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n! |
|
266 |
bind(L); // s (mod n) < n |
|
267 |
shldl(hi, lo); // x := x << s |
|
268 |
shll(lo); |
|
269 |
} |
|
270 |
||
271 |
||
272 |
void MacroAssembler::lshr(Register hi, Register lo, bool sign_extension) { |
|
273 |
// Java shift right long support (semantics as described in JVM spec., p.306 & p.310) |
|
274 |
// (basic idea for shift counts s >= n: x >> s == (x >> n) >> (s - n)) |
|
275 |
assert(hi != rcx, "must not use rcx"); |
|
276 |
assert(lo != rcx, "must not use rcx"); |
|
277 |
const Register s = rcx; // shift count |
|
278 |
const int n = BitsPerWord; |
|
279 |
Label L; |
|
280 |
andl(s, 0x3f); // s := s & 0x3f (s < 0x40) |
|
281 |
cmpl(s, n); // if (s < n) |
|
282 |
jcc(Assembler::less, L); // else (s >= n) |
|
283 |
movl(lo, hi); // x := x >> n |
|
284 |
if (sign_extension) sarl(hi, 31); |
|
285 |
else xorl(hi, hi); |
|
286 |
// Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n! |
|
287 |
bind(L); // s (mod n) < n |
|
288 |
shrdl(lo, hi); // x := x >> s |
|
289 |
if (sign_extension) sarl(hi); |
|
290 |
else shrl(hi); |
|
291 |
} |
|
292 |
||
293 |
void MacroAssembler::movoop(Register dst, jobject obj) { |
|
294 |
mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate()); |
|
295 |
} |
|
296 |
||
297 |
void MacroAssembler::movoop(Address dst, jobject obj) { |
|
298 |
mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate()); |
|
299 |
} |
|
300 |
||
301 |
void MacroAssembler::mov_metadata(Register dst, Metadata* obj) { |
|
302 |
mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate()); |
|
303 |
} |
|
304 |
||
305 |
void MacroAssembler::mov_metadata(Address dst, Metadata* obj) { |
|
306 |
mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate()); |
|
307 |
} |
|
308 |
||
23491 | 309 |
void MacroAssembler::movptr(Register dst, AddressLiteral src, Register scratch) { |
310 |
// scratch register is not used, |
|
311 |
// it is defined to match parameters of 64-bit version of this method. |
|
14626 | 312 |
if (src.is_lval()) { |
313 |
mov_literal32(dst, (intptr_t)src.target(), src.rspec()); |
|
314 |
} else { |
|
315 |
movl(dst, as_Address(src)); |
|
316 |
} |
|
317 |
} |
|
318 |
||
319 |
void MacroAssembler::movptr(ArrayAddress dst, Register src) { |
|
320 |
movl(as_Address(dst), src); |
|
321 |
} |
|
322 |
||
323 |
void MacroAssembler::movptr(Register dst, ArrayAddress src) { |
|
324 |
movl(dst, as_Address(src)); |
|
325 |
} |
|
326 |
||
327 |
// src should NEVER be a real pointer. Use AddressLiteral for true pointers |
|
328 |
void MacroAssembler::movptr(Address dst, intptr_t src) { |
|
329 |
movl(dst, src); |
|
330 |
} |
|
331 |
||
332 |
||
333 |
void MacroAssembler::pop_callee_saved_registers() { |
|
334 |
pop(rcx); |
|
335 |
pop(rdx); |
|
336 |
pop(rdi); |
|
337 |
pop(rsi); |
|
338 |
} |
|
339 |
||
340 |
void MacroAssembler::pop_fTOS() { |
|
341 |
fld_d(Address(rsp, 0)); |
|
342 |
addl(rsp, 2 * wordSize); |
|
343 |
} |
|
344 |
||
345 |
void MacroAssembler::push_callee_saved_registers() { |
|
346 |
push(rsi); |
|
347 |
push(rdi); |
|
348 |
push(rdx); |
|
349 |
push(rcx); |
|
350 |
} |
|
351 |
||
352 |
void MacroAssembler::push_fTOS() { |
|
353 |
subl(rsp, 2 * wordSize); |
|
354 |
fstp_d(Address(rsp, 0)); |
|
355 |
} |
|
356 |
||
357 |
||
358 |
void MacroAssembler::pushoop(jobject obj) { |
|
359 |
push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate()); |
|
360 |
} |
|
361 |
||
362 |
void MacroAssembler::pushklass(Metadata* obj) { |
|
363 |
push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate()); |
|
364 |
} |
|
365 |
||
366 |
void MacroAssembler::pushptr(AddressLiteral src) { |
|
367 |
if (src.is_lval()) { |
|
368 |
push_literal32((int32_t)src.target(), src.rspec()); |
|
369 |
} else { |
|
370 |
pushl(as_Address(src)); |
|
371 |
} |
|
372 |
} |
|
373 |
||
374 |
void MacroAssembler::set_word_if_not_zero(Register dst) { |
|
375 |
xorl(dst, dst); |
|
376 |
set_byte_if_not_zero(dst); |
|
377 |
} |
|
378 |
||
379 |
static void pass_arg0(MacroAssembler* masm, Register arg) { |
|
380 |
masm->push(arg); |
|
381 |
} |
|
382 |
||
383 |
static void pass_arg1(MacroAssembler* masm, Register arg) { |
|
384 |
masm->push(arg); |
|
385 |
} |
|
386 |
||
387 |
static void pass_arg2(MacroAssembler* masm, Register arg) { |
|
388 |
masm->push(arg); |
|
389 |
} |
|
390 |
||
391 |
static void pass_arg3(MacroAssembler* masm, Register arg) { |
|
392 |
masm->push(arg); |
|
393 |
} |
|
394 |
||
395 |
#ifndef PRODUCT |
|
396 |
extern "C" void findpc(intptr_t x); |
|
397 |
#endif |
|
398 |
||
399 |
void MacroAssembler::debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg) { |
|
400 |
// In order to get locks to work, we need to fake a in_VM state |
|
401 |
JavaThread* thread = JavaThread::current(); |
|
402 |
JavaThreadState saved_state = thread->thread_state(); |
|
403 |
thread->set_thread_state(_thread_in_vm); |
|
404 |
if (ShowMessageBoxOnError) { |
|
405 |
JavaThread* thread = JavaThread::current(); |
|
406 |
JavaThreadState saved_state = thread->thread_state(); |
|
407 |
thread->set_thread_state(_thread_in_vm); |
|
408 |
if (CountBytecodes || TraceBytecodes || StopInterpreterAt) { |
|
409 |
ttyLocker ttyl; |
|
410 |
BytecodeCounter::print(); |
|
411 |
} |
|
412 |
// To see where a verify_oop failed, get $ebx+40/X for this frame. |
|
413 |
// This is the value of eip which points to where verify_oop will return. |
|
414 |
if (os::message_box(msg, "Execution stopped, print registers?")) { |
|
415 |
print_state32(rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax, eip); |
|
416 |
BREAKPOINT; |
|
417 |
} |
|
418 |
} else { |
|
419 |
ttyLocker ttyl; |
|
420 |
::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n", msg); |
|
421 |
} |
|
422 |
// Don't assert holding the ttyLock |
|
33105
294e48b4f704
8080775: Better argument formatting for assert() and friends
david
parents:
32727
diff
changeset
|
423 |
assert(false, "DEBUG MESSAGE: %s", msg); |
14626 | 424 |
ThreadStateTransition::transition(thread, _thread_in_vm, saved_state); |
425 |
} |
|
426 |
||
427 |
void MacroAssembler::print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip) { |
|
428 |
ttyLocker ttyl; |
|
429 |
FlagSetting fs(Debugging, true); |
|
430 |
tty->print_cr("eip = 0x%08x", eip); |
|
431 |
#ifndef PRODUCT |
|
432 |
if ((WizardMode || Verbose) && PrintMiscellaneous) { |
|
433 |
tty->cr(); |
|
434 |
findpc(eip); |
|
435 |
tty->cr(); |
|
436 |
} |
|
437 |
#endif |
|
438 |
#define PRINT_REG(rax) \ |
|
439 |
{ tty->print("%s = ", #rax); os::print_location(tty, rax); } |
|
440 |
PRINT_REG(rax); |
|
441 |
PRINT_REG(rbx); |
|
442 |
PRINT_REG(rcx); |
|
443 |
PRINT_REG(rdx); |
|
444 |
PRINT_REG(rdi); |
|
445 |
PRINT_REG(rsi); |
|
446 |
PRINT_REG(rbp); |
|
447 |
PRINT_REG(rsp); |
|
448 |
#undef PRINT_REG |
|
449 |
// Print some words near top of staack. |
|
450 |
int* dump_sp = (int*) rsp; |
|
451 |
for (int col1 = 0; col1 < 8; col1++) { |
|
452 |
tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); |
|
453 |
os::print_location(tty, *dump_sp++); |
|
454 |
} |
|
455 |
for (int row = 0; row < 16; row++) { |
|
456 |
tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); |
|
457 |
for (int col = 0; col < 8; col++) { |
|
458 |
tty->print(" 0x%08x", *dump_sp++); |
|
459 |
} |
|
460 |
tty->cr(); |
|
461 |
} |
|
462 |
// Print some instructions around pc: |
|
463 |
Disassembler::decode((address)eip-64, (address)eip); |
|
464 |
tty->print_cr("--------"); |
|
465 |
Disassembler::decode((address)eip, (address)eip+32); |
|
466 |
} |
|
467 |
||
468 |
void MacroAssembler::stop(const char* msg) { |
|
469 |
ExternalAddress message((address)msg); |
|
470 |
// push address of message |
|
471 |
pushptr(message.addr()); |
|
472 |
{ Label L; call(L, relocInfo::none); bind(L); } // push eip |
|
473 |
pusha(); // push registers |
|
474 |
call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32))); |
|
475 |
hlt(); |
|
476 |
} |
|
477 |
||
478 |
void MacroAssembler::warn(const char* msg) { |
|
479 |
push_CPU_state(); |
|
480 |
||
481 |
ExternalAddress message((address) msg); |
|
482 |
// push address of message |
|
483 |
pushptr(message.addr()); |
|
484 |
||
485 |
call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning))); |
|
486 |
addl(rsp, wordSize); // discard argument |
|
487 |
pop_CPU_state(); |
|
488 |
} |
|
489 |
||
490 |
void MacroAssembler::print_state() { |
|
491 |
{ Label L; call(L, relocInfo::none); bind(L); } // push eip |
|
492 |
pusha(); // push registers |
|
493 |
||
494 |
push_CPU_state(); |
|
495 |
call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::print_state32))); |
|
496 |
pop_CPU_state(); |
|
497 |
||
498 |
popa(); |
|
499 |
addl(rsp, wordSize); |
|
500 |
} |
|
501 |
||
502 |
#else // _LP64 |
|
503 |
||
504 |
// 64 bit versions |
|
505 |
||
506 |
Address MacroAssembler::as_Address(AddressLiteral adr) { |
|
507 |
// amd64 always does this as a pc-rel |
|
508 |
// we can be absolute or disp based on the instruction type |
|
509 |
// jmp/call are displacements others are absolute |
|
510 |
assert(!adr.is_lval(), "must be rval"); |
|
511 |
assert(reachable(adr), "must be"); |
|
512 |
return Address((int32_t)(intptr_t)(adr.target() - pc()), adr.target(), adr.reloc()); |
|
513 |
||
514 |
} |
|
515 |
||
516 |
Address MacroAssembler::as_Address(ArrayAddress adr) { |
|
517 |
AddressLiteral base = adr.base(); |
|
518 |
lea(rscratch1, base); |
|
519 |
Address index = adr.index(); |
|
520 |
assert(index._disp == 0, "must not have disp"); // maybe it can? |
|
521 |
Address array(rscratch1, index._index, index._scale, index._disp); |
|
522 |
return array; |
|
523 |
} |
|
524 |
||
525 |
void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) { |
|
526 |
Label L, E; |
|
527 |
||
528 |
#ifdef _WIN64 |
|
529 |
// Windows always allocates space for it's register args |
|
530 |
assert(num_args <= 4, "only register arguments supported"); |
|
531 |
subq(rsp, frame::arg_reg_save_area_bytes); |
|
532 |
#endif |
|
533 |
||
534 |
// Align stack if necessary |
|
535 |
testl(rsp, 15); |
|
536 |
jcc(Assembler::zero, L); |
|
537 |
||
538 |
subq(rsp, 8); |
|
539 |
{ |
|
540 |
call(RuntimeAddress(entry_point)); |
|
541 |
} |
|
542 |
addq(rsp, 8); |
|
543 |
jmp(E); |
|
544 |
||
545 |
bind(L); |
|
546 |
{ |
|
547 |
call(RuntimeAddress(entry_point)); |
|
548 |
} |
|
549 |
||
550 |
bind(E); |
|
551 |
||
552 |
#ifdef _WIN64 |
|
553 |
// restore stack pointer |
|
554 |
addq(rsp, frame::arg_reg_save_area_bytes); |
|
555 |
#endif |
|
556 |
||
557 |
} |
|
558 |
||
559 |
void MacroAssembler::cmp64(Register src1, AddressLiteral src2) { |
|
560 |
assert(!src2.is_lval(), "should use cmpptr"); |
|
561 |
||
562 |
if (reachable(src2)) { |
|
563 |
cmpq(src1, as_Address(src2)); |
|
564 |
} else { |
|
565 |
lea(rscratch1, src2); |
|
566 |
Assembler::cmpq(src1, Address(rscratch1, 0)); |
|
567 |
} |
|
568 |
} |
|
569 |
||
570 |
int MacroAssembler::corrected_idivq(Register reg) { |
|
571 |
// Full implementation of Java ldiv and lrem; checks for special |
|
572 |
// case as described in JVM spec., p.243 & p.271. The function |
|
573 |
// returns the (pc) offset of the idivl instruction - may be needed |
|
574 |
// for implicit exceptions. |
|
575 |
// |
|
576 |
// normal case special case |
|
577 |
// |
|
578 |
// input : rax: dividend min_long |
|
579 |
// reg: divisor (may not be eax/edx) -1 |
|
580 |
// |
|
581 |
// output: rax: quotient (= rax idiv reg) min_long |
|
582 |
// rdx: remainder (= rax irem reg) 0 |
|
583 |
assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register"); |
|
584 |
static const int64_t min_long = 0x8000000000000000; |
|
585 |
Label normal_case, special_case; |
|
586 |
||
587 |
// check for special case |
|
588 |
cmp64(rax, ExternalAddress((address) &min_long)); |
|
589 |
jcc(Assembler::notEqual, normal_case); |
|
590 |
xorl(rdx, rdx); // prepare rdx for possible special case (where |
|
591 |
// remainder = 0) |
|
592 |
cmpq(reg, -1); |
|
593 |
jcc(Assembler::equal, special_case); |
|
594 |
||
595 |
// handle normal case |
|
596 |
bind(normal_case); |
|
597 |
cdqq(); |
|
598 |
int idivq_offset = offset(); |
|
599 |
idivq(reg); |
|
600 |
||
601 |
// normal and special case exit |
|
602 |
bind(special_case); |
|
603 |
||
604 |
return idivq_offset; |
|
605 |
} |
|
606 |
||
607 |
void MacroAssembler::decrementq(Register reg, int value) { |
|
608 |
if (value == min_jint) { subq(reg, value); return; } |
|
609 |
if (value < 0) { incrementq(reg, -value); return; } |
|
610 |
if (value == 0) { ; return; } |
|
611 |
if (value == 1 && UseIncDec) { decq(reg) ; return; } |
|
612 |
/* else */ { subq(reg, value) ; return; } |
|
613 |
} |
|
614 |
||
615 |
void MacroAssembler::decrementq(Address dst, int value) { |
|
616 |
if (value == min_jint) { subq(dst, value); return; } |
|
617 |
if (value < 0) { incrementq(dst, -value); return; } |
|
618 |
if (value == 0) { ; return; } |
|
619 |
if (value == 1 && UseIncDec) { decq(dst) ; return; } |
|
620 |
/* else */ { subq(dst, value) ; return; } |
|
621 |
} |
|
622 |
||
23491 | 623 |
void MacroAssembler::incrementq(AddressLiteral dst) { |
624 |
if (reachable(dst)) { |
|
625 |
incrementq(as_Address(dst)); |
|
626 |
} else { |
|
627 |
lea(rscratch1, dst); |
|
628 |
incrementq(Address(rscratch1, 0)); |
|
629 |
} |
|
630 |
} |
|
631 |
||
14626 | 632 |
void MacroAssembler::incrementq(Register reg, int value) { |
633 |
if (value == min_jint) { addq(reg, value); return; } |
|
634 |
if (value < 0) { decrementq(reg, -value); return; } |
|
635 |
if (value == 0) { ; return; } |
|
636 |
if (value == 1 && UseIncDec) { incq(reg) ; return; } |
|
637 |
/* else */ { addq(reg, value) ; return; } |
|
638 |
} |
|
639 |
||
640 |
void MacroAssembler::incrementq(Address dst, int value) { |
|
641 |
if (value == min_jint) { addq(dst, value); return; } |
|
642 |
if (value < 0) { decrementq(dst, -value); return; } |
|
643 |
if (value == 0) { ; return; } |
|
644 |
if (value == 1 && UseIncDec) { incq(dst) ; return; } |
|
645 |
/* else */ { addq(dst, value) ; return; } |
|
646 |
} |
|
647 |
||
648 |
// 32bit can do a case table jump in one instruction but we no longer allow the base |
|
649 |
// to be installed in the Address class |
|
650 |
void MacroAssembler::jump(ArrayAddress entry) { |
|
651 |
lea(rscratch1, entry.base()); |
|
652 |
Address dispatch = entry.index(); |
|
653 |
assert(dispatch._base == noreg, "must be"); |
|
654 |
dispatch._base = rscratch1; |
|
655 |
jmp(dispatch); |
|
656 |
} |
|
657 |
||
658 |
void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) { |
|
659 |
ShouldNotReachHere(); // 64bit doesn't use two regs |
|
660 |
cmpq(x_lo, y_lo); |
|
661 |
} |
|
662 |
||
663 |
void MacroAssembler::lea(Register dst, AddressLiteral src) { |
|
664 |
mov_literal64(dst, (intptr_t)src.target(), src.rspec()); |
|
665 |
} |
|
666 |
||
667 |
void MacroAssembler::lea(Address dst, AddressLiteral adr) { |
|
668 |
mov_literal64(rscratch1, (intptr_t)adr.target(), adr.rspec()); |
|
669 |
movptr(dst, rscratch1); |
|
670 |
} |
|
671 |
||
672 |
void MacroAssembler::leave() { |
|
673 |
// %%% is this really better? Why not on 32bit too? |
|
14837
a75c3082d106
8004250: replace AbstractAssembler a_byte/a_long with emit_int8/emit_int32
twisti
parents:
14834
diff
changeset
|
674 |
emit_int8((unsigned char)0xC9); // LEAVE |
14626 | 675 |
} |
676 |
||
677 |
void MacroAssembler::lneg(Register hi, Register lo) { |
|
678 |
ShouldNotReachHere(); // 64bit doesn't use two regs |
|
679 |
negq(lo); |
|
680 |
} |
|
681 |
||
682 |
void MacroAssembler::movoop(Register dst, jobject obj) { |
|
683 |
mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate()); |
|
684 |
} |
|
685 |
||
686 |
void MacroAssembler::movoop(Address dst, jobject obj) { |
|
687 |
mov_literal64(rscratch1, (intptr_t)obj, oop_Relocation::spec_for_immediate()); |
|
688 |
movq(dst, rscratch1); |
|
689 |
} |
|
690 |
||
691 |
void MacroAssembler::mov_metadata(Register dst, Metadata* obj) { |
|
692 |
mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate()); |
|
693 |
} |
|
694 |
||
695 |
void MacroAssembler::mov_metadata(Address dst, Metadata* obj) { |
|
696 |
mov_literal64(rscratch1, (intptr_t)obj, metadata_Relocation::spec_for_immediate()); |
|
697 |
movq(dst, rscratch1); |
|
698 |
} |
|
699 |
||
23491 | 700 |
void MacroAssembler::movptr(Register dst, AddressLiteral src, Register scratch) { |
14626 | 701 |
if (src.is_lval()) { |
702 |
mov_literal64(dst, (intptr_t)src.target(), src.rspec()); |
|
703 |
} else { |
|
704 |
if (reachable(src)) { |
|
705 |
movq(dst, as_Address(src)); |
|
706 |
} else { |
|
23491 | 707 |
lea(scratch, src); |
708 |
movq(dst, Address(scratch, 0)); |
|
14626 | 709 |
} |
710 |
} |
|
711 |
} |
|
712 |
||
713 |
void MacroAssembler::movptr(ArrayAddress dst, Register src) { |
|
714 |
movq(as_Address(dst), src); |
|
715 |
} |
|
716 |
||
717 |
void MacroAssembler::movptr(Register dst, ArrayAddress src) { |
|
718 |
movq(dst, as_Address(src)); |
|
719 |
} |
|
720 |
||
721 |
// src should NEVER be a real pointer. Use AddressLiteral for true pointers |
|
722 |
void MacroAssembler::movptr(Address dst, intptr_t src) { |
|
723 |
mov64(rscratch1, src); |
|
724 |
movq(dst, rscratch1); |
|
725 |
} |
|
726 |
||
727 |
// These are mostly for initializing NULL |
|
728 |
void MacroAssembler::movptr(Address dst, int32_t src) { |
|
729 |
movslq(dst, src); |
|
730 |
} |
|
731 |
||
732 |
void MacroAssembler::movptr(Register dst, int32_t src) { |
|
733 |
mov64(dst, (intptr_t)src); |
|
734 |
} |
|
735 |
||
736 |
void MacroAssembler::pushoop(jobject obj) { |
|
737 |
movoop(rscratch1, obj); |
|
738 |
push(rscratch1); |
|
739 |
} |
|
740 |
||
741 |
void MacroAssembler::pushklass(Metadata* obj) { |
|
742 |
mov_metadata(rscratch1, obj); |
|
743 |
push(rscratch1); |
|
744 |
} |
|
745 |
||
746 |
void MacroAssembler::pushptr(AddressLiteral src) { |
|
747 |
lea(rscratch1, src); |
|
748 |
if (src.is_lval()) { |
|
749 |
push(rscratch1); |
|
750 |
} else { |
|
751 |
pushq(Address(rscratch1, 0)); |
|
752 |
} |
|
753 |
} |
|
754 |
||
755 |
void MacroAssembler::reset_last_Java_frame(bool clear_fp, |
|
756 |
bool clear_pc) { |
|
757 |
// we must set sp to zero to clear frame |
|
758 |
movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), NULL_WORD); |
|
759 |
// must clear fp, so that compiled frames are not confused; it is |
|
760 |
// possible that we need it only for debugging |
|
761 |
if (clear_fp) { |
|
762 |
movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), NULL_WORD); |
|
763 |
} |
|
764 |
||
765 |
if (clear_pc) { |
|
766 |
movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), NULL_WORD); |
|
767 |
} |
|
768 |
} |
|
769 |
||
770 |
void MacroAssembler::set_last_Java_frame(Register last_java_sp, |
|
771 |
Register last_java_fp, |
|
772 |
address last_java_pc) { |
|
773 |
// determine last_java_sp register |
|
774 |
if (!last_java_sp->is_valid()) { |
|
775 |
last_java_sp = rsp; |
|
776 |
} |
|
777 |
||
778 |
// last_java_fp is optional |
|
779 |
if (last_java_fp->is_valid()) { |
|
780 |
movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), |
|
781 |
last_java_fp); |
|
782 |
} |
|
783 |
||
784 |
// last_java_pc is optional |
|
785 |
if (last_java_pc != NULL) { |
|
786 |
Address java_pc(r15_thread, |
|
787 |
JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset()); |
|
788 |
lea(rscratch1, InternalAddress(last_java_pc)); |
|
789 |
movptr(java_pc, rscratch1); |
|
790 |
} |
|
791 |
||
792 |
movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), last_java_sp); |
|
793 |
} |
|
794 |
||
795 |
static void pass_arg0(MacroAssembler* masm, Register arg) { |
|
796 |
if (c_rarg0 != arg ) { |
|
797 |
masm->mov(c_rarg0, arg); |
|
798 |
} |
|
799 |
} |
|
800 |
||
801 |
static void pass_arg1(MacroAssembler* masm, Register arg) { |
|
802 |
if (c_rarg1 != arg ) { |
|
803 |
masm->mov(c_rarg1, arg); |
|
804 |
} |
|
805 |
} |
|
806 |
||
807 |
static void pass_arg2(MacroAssembler* masm, Register arg) { |
|
808 |
if (c_rarg2 != arg ) { |
|
809 |
masm->mov(c_rarg2, arg); |
|
810 |
} |
|
811 |
} |
|
812 |
||
813 |
static void pass_arg3(MacroAssembler* masm, Register arg) { |
|
814 |
if (c_rarg3 != arg ) { |
|
815 |
masm->mov(c_rarg3, arg); |
|
816 |
} |
|
817 |
} |
|
818 |
||
819 |
void MacroAssembler::stop(const char* msg) { |
|
820 |
address rip = pc(); |
|
821 |
pusha(); // get regs on stack |
|
822 |
lea(c_rarg0, ExternalAddress((address) msg)); |
|
823 |
lea(c_rarg1, InternalAddress(rip)); |
|
824 |
movq(c_rarg2, rsp); // pass pointer to regs array |
|
825 |
andq(rsp, -16); // align stack as required by ABI |
|
826 |
call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64))); |
|
827 |
hlt(); |
|
828 |
} |
|
829 |
||
830 |
void MacroAssembler::warn(const char* msg) { |
|
831 |
push(rbp); |
|
832 |
movq(rbp, rsp); |
|
833 |
andq(rsp, -16); // align stack as required by push_CPU_state and call |
|
834 |
push_CPU_state(); // keeps alignment at 16 bytes |
|
835 |
lea(c_rarg0, ExternalAddress((address) msg)); |
|
836 |
call_VM_leaf(CAST_FROM_FN_PTR(address, warning), c_rarg0); |
|
837 |
pop_CPU_state(); |
|
838 |
mov(rsp, rbp); |
|
839 |
pop(rbp); |
|
840 |
} |
|
841 |
||
842 |
void MacroAssembler::print_state() { |
|
843 |
address rip = pc(); |
|
844 |
pusha(); // get regs on stack |
|
845 |
push(rbp); |
|
846 |
movq(rbp, rsp); |
|
847 |
andq(rsp, -16); // align stack as required by push_CPU_state and call |
|
848 |
push_CPU_state(); // keeps alignment at 16 bytes |
|
849 |
||
850 |
lea(c_rarg0, InternalAddress(rip)); |
|
851 |
lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array |
|
852 |
call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1); |
|
853 |
||
854 |
pop_CPU_state(); |
|
855 |
mov(rsp, rbp); |
|
856 |
pop(rbp); |
|
857 |
popa(); |
|
858 |
} |
|
859 |
||
860 |
#ifndef PRODUCT |
|
861 |
extern "C" void findpc(intptr_t x); |
|
862 |
#endif |
|
863 |
||
864 |
void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) { |
|
865 |
// In order to get locks to work, we need to fake a in_VM state |
|
866 |
if (ShowMessageBoxOnError) { |
|
867 |
JavaThread* thread = JavaThread::current(); |
|
868 |
JavaThreadState saved_state = thread->thread_state(); |
|
869 |
thread->set_thread_state(_thread_in_vm); |
|
870 |
#ifndef PRODUCT |
|
871 |
if (CountBytecodes || TraceBytecodes || StopInterpreterAt) { |
|
872 |
ttyLocker ttyl; |
|
873 |
BytecodeCounter::print(); |
|
874 |
} |
|
875 |
#endif |
|
876 |
// To see where a verify_oop failed, get $ebx+40/X for this frame. |
|
877 |
// XXX correct this offset for amd64 |
|
878 |
// This is the value of eip which points to where verify_oop will return. |
|
879 |
if (os::message_box(msg, "Execution stopped, print registers?")) { |
|
880 |
print_state64(pc, regs); |
|
881 |
BREAKPOINT; |
|
882 |
assert(false, "start up GDB"); |
|
883 |
} |
|
884 |
ThreadStateTransition::transition(thread, _thread_in_vm, saved_state); |
|
885 |
} else { |
|
886 |
ttyLocker ttyl; |
|
887 |
::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n", |
|
888 |
msg); |
|
33105
294e48b4f704
8080775: Better argument formatting for assert() and friends
david
parents:
32727
diff
changeset
|
889 |
assert(false, "DEBUG MESSAGE: %s", msg); |
14626 | 890 |
} |
891 |
} |
|
892 |
||
893 |
void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) { |
|
894 |
ttyLocker ttyl; |
|
895 |
FlagSetting fs(Debugging, true); |
|
896 |
tty->print_cr("rip = 0x%016lx", pc); |
|
897 |
#ifndef PRODUCT |
|
898 |
tty->cr(); |
|
899 |
findpc(pc); |
|
900 |
tty->cr(); |
|
901 |
#endif |
|
902 |
#define PRINT_REG(rax, value) \ |
|
903 |
{ tty->print("%s = ", #rax); os::print_location(tty, value); } |
|
904 |
PRINT_REG(rax, regs[15]); |
|
905 |
PRINT_REG(rbx, regs[12]); |
|
906 |
PRINT_REG(rcx, regs[14]); |
|
907 |
PRINT_REG(rdx, regs[13]); |
|
908 |
PRINT_REG(rdi, regs[8]); |
|
909 |
PRINT_REG(rsi, regs[9]); |
|
910 |
PRINT_REG(rbp, regs[10]); |
|
911 |
PRINT_REG(rsp, regs[11]); |
|
912 |
PRINT_REG(r8 , regs[7]); |
|
913 |
PRINT_REG(r9 , regs[6]); |
|
914 |
PRINT_REG(r10, regs[5]); |
|
915 |
PRINT_REG(r11, regs[4]); |
|
916 |
PRINT_REG(r12, regs[3]); |
|
917 |
PRINT_REG(r13, regs[2]); |
|
918 |
PRINT_REG(r14, regs[1]); |
|
919 |
PRINT_REG(r15, regs[0]); |
|
920 |
#undef PRINT_REG |
|
921 |
// Print some words near top of staack. |
|
922 |
int64_t* rsp = (int64_t*) regs[11]; |
|
923 |
int64_t* dump_sp = rsp; |
|
924 |
for (int col1 = 0; col1 < 8; col1++) { |
|
925 |
tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (int64_t)dump_sp); |
|
926 |
os::print_location(tty, *dump_sp++); |
|
927 |
} |
|
928 |
for (int row = 0; row < 25; row++) { |
|
929 |
tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (int64_t)dump_sp); |
|
930 |
for (int col = 0; col < 4; col++) { |
|
931 |
tty->print(" 0x%016lx", *dump_sp++); |
|
932 |
} |
|
933 |
tty->cr(); |
|
934 |
} |
|
935 |
// Print some instructions around pc: |
|
936 |
Disassembler::decode((address)pc-64, (address)pc); |
|
937 |
tty->print_cr("--------"); |
|
938 |
Disassembler::decode((address)pc, (address)pc+32); |
|
939 |
} |
|
940 |
||
941 |
#endif // _LP64 |
|
942 |
||
943 |
// Now versions that are common to 32/64 bit |
|
944 |
||
945 |
void MacroAssembler::addptr(Register dst, int32_t imm32) { |
|
946 |
LP64_ONLY(addq(dst, imm32)) NOT_LP64(addl(dst, imm32)); |
|
947 |
} |
|
948 |
||
949 |
void MacroAssembler::addptr(Register dst, Register src) { |
|
950 |
LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src)); |
|
951 |
} |
|
952 |
||
953 |
void MacroAssembler::addptr(Address dst, Register src) { |
|
954 |
LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src)); |
|
955 |
} |
|
956 |
||
957 |
void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src) { |
|
958 |
if (reachable(src)) { |
|
959 |
Assembler::addsd(dst, as_Address(src)); |
|
960 |
} else { |
|
961 |
lea(rscratch1, src); |
|
962 |
Assembler::addsd(dst, Address(rscratch1, 0)); |
|
963 |
} |
|
964 |
} |
|
965 |
||
966 |
void MacroAssembler::addss(XMMRegister dst, AddressLiteral src) { |
|
967 |
if (reachable(src)) { |
|
968 |
addss(dst, as_Address(src)); |
|
969 |
} else { |
|
970 |
lea(rscratch1, src); |
|
971 |
addss(dst, Address(rscratch1, 0)); |
|
972 |
} |
|
973 |
} |
|
974 |
||
35540
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
975 |
void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src) { |
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
976 |
if (reachable(src)) { |
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
977 |
Assembler::addpd(dst, as_Address(src)); |
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
978 |
} else { |
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
979 |
lea(rscratch1, src); |
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
980 |
Assembler::addpd(dst, Address(rscratch1, 0)); |
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
981 |
} |
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
982 |
} |
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
983 |
|
14626 | 984 |
void MacroAssembler::align(int modulus) { |
32203 | 985 |
align(modulus, offset()); |
986 |
} |
|
987 |
||
988 |
void MacroAssembler::align(int modulus, int target) { |
|
989 |
if (target % modulus != 0) { |
|
990 |
nop(modulus - (target % modulus)); |
|
14626 | 991 |
} |
992 |
} |
|
993 |
||
994 |
void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src) { |
|
995 |
// Used in sign-masking with aligned address. |
|
996 |
assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); |
|
997 |
if (reachable(src)) { |
|
998 |
Assembler::andpd(dst, as_Address(src)); |
|
999 |
} else { |
|
1000 |
lea(rscratch1, src); |
|
1001 |
Assembler::andpd(dst, Address(rscratch1, 0)); |
|
1002 |
} |
|
1003 |
} |
|
1004 |
||
1005 |
void MacroAssembler::andps(XMMRegister dst, AddressLiteral src) { |
|
1006 |
// Used in sign-masking with aligned address. |
|
1007 |
assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); |
|
1008 |
if (reachable(src)) { |
|
1009 |
Assembler::andps(dst, as_Address(src)); |
|
1010 |
} else { |
|
1011 |
lea(rscratch1, src); |
|
1012 |
Assembler::andps(dst, Address(rscratch1, 0)); |
|
1013 |
} |
|
1014 |
} |
|
1015 |
||
1016 |
void MacroAssembler::andptr(Register dst, int32_t imm32) { |
|
1017 |
LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32)); |
|
1018 |
} |
|
1019 |
||
23491 | 1020 |
void MacroAssembler::atomic_incl(Address counter_addr) { |
1021 |
if (os::is_MP()) |
|
1022 |
lock(); |
|
1023 |
incrementl(counter_addr); |
|
1024 |
} |
|
1025 |
||
1026 |
void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register scr) { |
|
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1027 |
if (reachable(counter_addr)) { |
23491 | 1028 |
atomic_incl(as_Address(counter_addr)); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1029 |
} else { |
23491 | 1030 |
lea(scr, counter_addr); |
1031 |
atomic_incl(Address(scr, 0)); |
|
1032 |
} |
|
1033 |
} |
|
1034 |
||
1035 |
#ifdef _LP64 |
|
1036 |
void MacroAssembler::atomic_incq(Address counter_addr) { |
|
1037 |
if (os::is_MP()) |
|
1038 |
lock(); |
|
1039 |
incrementq(counter_addr); |
|
1040 |
} |
|
1041 |
||
1042 |
void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register scr) { |
|
1043 |
if (reachable(counter_addr)) { |
|
1044 |
atomic_incq(as_Address(counter_addr)); |
|
1045 |
} else { |
|
1046 |
lea(scr, counter_addr); |
|
1047 |
atomic_incq(Address(scr, 0)); |
|
1048 |
} |
|
1049 |
} |
|
1050 |
#endif |
|
14626 | 1051 |
|
1052 |
// Writes to stack successive pages until offset reached to check for |
|
1053 |
// stack overflow + shadow pages. This clobbers tmp. |
|
1054 |
void MacroAssembler::bang_stack_size(Register size, Register tmp) { |
|
1055 |
movptr(tmp, rsp); |
|
1056 |
// Bang stack for total size given plus shadow page size. |
|
1057 |
// Bang one page at a time because large size can bang beyond yellow and |
|
1058 |
// red zones. |
|
1059 |
Label loop; |
|
1060 |
bind(loop); |
|
1061 |
movl(Address(tmp, (-os::vm_page_size())), size ); |
|
1062 |
subptr(tmp, os::vm_page_size()); |
|
1063 |
subl(size, os::vm_page_size()); |
|
1064 |
jcc(Assembler::greater, loop); |
|
1065 |
||
1066 |
// Bang down shadow pages too. |
|
21528
479228ecf6ac
8026775: nsk/jvmti/RedefineClasses/StressRedefine crashes due to EXCEPTION_ACCESS_VIOLATION
mikael
parents:
21188
diff
changeset
|
1067 |
// At this point, (tmp-0) is the last address touched, so don't |
479228ecf6ac
8026775: nsk/jvmti/RedefineClasses/StressRedefine crashes due to EXCEPTION_ACCESS_VIOLATION
mikael
parents:
21188
diff
changeset
|
1068 |
// touch it again. (It was touched as (tmp-pagesize) but then tmp |
479228ecf6ac
8026775: nsk/jvmti/RedefineClasses/StressRedefine crashes due to EXCEPTION_ACCESS_VIOLATION
mikael
parents:
21188
diff
changeset
|
1069 |
// was post-decremented.) Skip this address by starting at i=1, and |
479228ecf6ac
8026775: nsk/jvmti/RedefineClasses/StressRedefine crashes due to EXCEPTION_ACCESS_VIOLATION
mikael
parents:
21188
diff
changeset
|
1070 |
// touch a few more pages below. N.B. It is important to touch all |
35201
996db89f378e
8139864: Improve handling of stack protection zones.
goetz
parents:
35071
diff
changeset
|
1071 |
// the way down including all pages in the shadow zone. |
996db89f378e
8139864: Improve handling of stack protection zones.
goetz
parents:
35071
diff
changeset
|
1072 |
for (int i = 1; i < ((int)JavaThread::stack_shadow_zone_size() / os::vm_page_size()); i++) { |
14626 | 1073 |
// this could be any sized move but this is can be a debugging crumb |
1074 |
// so the bigger the better. |
|
1075 |
movptr(Address(tmp, (-i*os::vm_page_size())), size ); |
|
1076 |
} |
|
1077 |
} |
|
1078 |
||
35071
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1079 |
void MacroAssembler::reserved_stack_check() { |
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1080 |
// testing if reserved zone needs to be enabled |
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1081 |
Label no_reserved_zone_enabling; |
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1082 |
Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread); |
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1083 |
NOT_LP64(get_thread(rsi);) |
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1084 |
|
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1085 |
cmpptr(rsp, Address(thread, JavaThread::reserved_stack_activation_offset())); |
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1086 |
jcc(Assembler::below, no_reserved_zone_enabling); |
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1087 |
|
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1088 |
call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), thread); |
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1089 |
jump(RuntimeAddress(StubRoutines::throw_delayed_StackOverflowError_entry())); |
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1090 |
should_not_reach_here(); |
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1091 |
|
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1092 |
bind(no_reserved_zone_enabling); |
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1093 |
} |
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1094 |
|
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1095 |
int MacroAssembler::biased_locking_enter(Register lock_reg, |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1096 |
Register obj_reg, |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1097 |
Register swap_reg, |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1098 |
Register tmp_reg, |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1099 |
bool swap_reg_contains_mark, |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1100 |
Label& done, |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1101 |
Label* slow_case, |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1102 |
BiasedLockingCounters* counters) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1103 |
assert(UseBiasedLocking, "why call this otherwise?"); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1104 |
assert(swap_reg == rax, "swap_reg must be rax for cmpxchgq"); |
30749
39a2475280ee
6811960: x86 biasedlocking epoch expired rare bug
mockner
parents:
30310
diff
changeset
|
1105 |
assert(tmp_reg != noreg, "tmp_reg must be supplied"); |
39a2475280ee
6811960: x86 biasedlocking epoch expired rare bug
mockner
parents:
30310
diff
changeset
|
1106 |
assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1107 |
assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout"); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1108 |
Address mark_addr (obj_reg, oopDesc::mark_offset_in_bytes()); |
37251
9fc139ad74b5
8152358: code and comment cleanups found during the hunt for 8077392
dcubed
parents:
36561
diff
changeset
|
1109 |
NOT_LP64( Address saved_mark_addr(lock_reg, 0); ) |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1110 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1111 |
if (PrintBiasedLockingStatistics && counters == NULL) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1112 |
counters = BiasedLocking::counters(); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1113 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1114 |
// Biased locking |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1115 |
// See whether the lock is currently biased toward our thread and |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1116 |
// whether the epoch is still valid |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1117 |
// Note that the runtime guarantees sufficient alignment of JavaThread |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1118 |
// pointers to allow age to be placed into low bits |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1119 |
// First check to see whether biasing is even enabled for this object |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1120 |
Label cas_label; |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1121 |
int null_check_offset = -1; |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1122 |
if (!swap_reg_contains_mark) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1123 |
null_check_offset = offset(); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1124 |
movptr(swap_reg, mark_addr); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1125 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1126 |
movptr(tmp_reg, swap_reg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1127 |
andptr(tmp_reg, markOopDesc::biased_lock_mask_in_place); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1128 |
cmpptr(tmp_reg, markOopDesc::biased_lock_pattern); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1129 |
jcc(Assembler::notEqual, cas_label); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1130 |
// The bias pattern is present in the object's header. Need to check |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1131 |
// whether the bias owner and the epoch are both still current. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1132 |
#ifndef _LP64 |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1133 |
// Note that because there is no current thread register on x86_32 we |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1134 |
// need to store off the mark word we read out of the object to |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1135 |
// avoid reloading it and needing to recheck invariants below. This |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1136 |
// store is unfortunate but it makes the overall code shorter and |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1137 |
// simpler. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1138 |
movptr(saved_mark_addr, swap_reg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1139 |
#endif |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1140 |
if (swap_reg_contains_mark) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1141 |
null_check_offset = offset(); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1142 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1143 |
load_prototype_header(tmp_reg, obj_reg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1144 |
#ifdef _LP64 |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1145 |
orptr(tmp_reg, r15_thread); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1146 |
xorptr(tmp_reg, swap_reg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1147 |
Register header_reg = tmp_reg; |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1148 |
#else |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1149 |
xorptr(tmp_reg, swap_reg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1150 |
get_thread(swap_reg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1151 |
xorptr(swap_reg, tmp_reg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1152 |
Register header_reg = swap_reg; |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1153 |
#endif |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1154 |
andptr(header_reg, ~((int) markOopDesc::age_mask_in_place)); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1155 |
if (counters != NULL) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1156 |
cond_inc32(Assembler::zero, |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1157 |
ExternalAddress((address) counters->biased_lock_entry_count_addr())); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1158 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1159 |
jcc(Assembler::equal, done); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1160 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1161 |
Label try_revoke_bias; |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1162 |
Label try_rebias; |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1163 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1164 |
// At this point we know that the header has the bias pattern and |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1165 |
// that we are not the bias owner in the current epoch. We need to |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1166 |
// figure out more details about the state of the header in order to |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1167 |
// know what operations can be legally performed on the object's |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1168 |
// header. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1169 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1170 |
// If the low three bits in the xor result aren't clear, that means |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1171 |
// the prototype header is no longer biased and we have to revoke |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1172 |
// the bias on this object. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1173 |
testptr(header_reg, markOopDesc::biased_lock_mask_in_place); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1174 |
jccb(Assembler::notZero, try_revoke_bias); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1175 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1176 |
// Biasing is still enabled for this data type. See whether the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1177 |
// epoch of the current bias is still valid, meaning that the epoch |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1178 |
// bits of the mark word are equal to the epoch bits of the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1179 |
// prototype header. (Note that the prototype header's epoch bits |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1180 |
// only change at a safepoint.) If not, attempt to rebias the object |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1181 |
// toward the current thread. Note that we must be absolutely sure |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1182 |
// that the current epoch is invalid in order to do this because |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1183 |
// otherwise the manipulations it performs on the mark word are |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1184 |
// illegal. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1185 |
testptr(header_reg, markOopDesc::epoch_mask_in_place); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1186 |
jccb(Assembler::notZero, try_rebias); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1187 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1188 |
// The epoch of the current bias is still valid but we know nothing |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1189 |
// about the owner; it might be set or it might be clear. Try to |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1190 |
// acquire the bias of the object using an atomic operation. If this |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1191 |
// fails we will go in to the runtime to revoke the object's bias. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1192 |
// Note that we first construct the presumed unbiased header so we |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1193 |
// don't accidentally blow away another thread's valid bias. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1194 |
NOT_LP64( movptr(swap_reg, saved_mark_addr); ) |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1195 |
andptr(swap_reg, |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1196 |
markOopDesc::biased_lock_mask_in_place | markOopDesc::age_mask_in_place | markOopDesc::epoch_mask_in_place); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1197 |
#ifdef _LP64 |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1198 |
movptr(tmp_reg, swap_reg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1199 |
orptr(tmp_reg, r15_thread); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1200 |
#else |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1201 |
get_thread(tmp_reg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1202 |
orptr(tmp_reg, swap_reg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1203 |
#endif |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1204 |
if (os::is_MP()) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1205 |
lock(); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1206 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1207 |
cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1208 |
// If the biasing toward our thread failed, this means that |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1209 |
// another thread succeeded in biasing it toward itself and we |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1210 |
// need to revoke that bias. The revocation will occur in the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1211 |
// interpreter runtime in the slow case. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1212 |
if (counters != NULL) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1213 |
cond_inc32(Assembler::zero, |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1214 |
ExternalAddress((address) counters->anonymously_biased_lock_entry_count_addr())); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1215 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1216 |
if (slow_case != NULL) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1217 |
jcc(Assembler::notZero, *slow_case); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1218 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1219 |
jmp(done); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1220 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1221 |
bind(try_rebias); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1222 |
// At this point we know the epoch has expired, meaning that the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1223 |
// current "bias owner", if any, is actually invalid. Under these |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1224 |
// circumstances _only_, we are allowed to use the current header's |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1225 |
// value as the comparison value when doing the cas to acquire the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1226 |
// bias in the current epoch. In other words, we allow transfer of |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1227 |
// the bias from one thread to another directly in this situation. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1228 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1229 |
// FIXME: due to a lack of registers we currently blow away the age |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1230 |
// bits in this situation. Should attempt to preserve them. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1231 |
load_prototype_header(tmp_reg, obj_reg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1232 |
#ifdef _LP64 |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1233 |
orptr(tmp_reg, r15_thread); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1234 |
#else |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1235 |
get_thread(swap_reg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1236 |
orptr(tmp_reg, swap_reg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1237 |
movptr(swap_reg, saved_mark_addr); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1238 |
#endif |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1239 |
if (os::is_MP()) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1240 |
lock(); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1241 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1242 |
cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1243 |
// If the biasing toward our thread failed, then another thread |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1244 |
// succeeded in biasing it toward itself and we need to revoke that |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1245 |
// bias. The revocation will occur in the runtime in the slow case. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1246 |
if (counters != NULL) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1247 |
cond_inc32(Assembler::zero, |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1248 |
ExternalAddress((address) counters->rebiased_lock_entry_count_addr())); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1249 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1250 |
if (slow_case != NULL) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1251 |
jcc(Assembler::notZero, *slow_case); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1252 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1253 |
jmp(done); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1254 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1255 |
bind(try_revoke_bias); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1256 |
// The prototype mark in the klass doesn't have the bias bit set any |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1257 |
// more, indicating that objects of this data type are not supposed |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1258 |
// to be biased any more. We are going to try to reset the mark of |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1259 |
// this object to the prototype value and fall through to the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1260 |
// CAS-based locking scheme. Note that if our CAS fails, it means |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1261 |
// that another thread raced us for the privilege of revoking the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1262 |
// bias of this particular object, so it's okay to continue in the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1263 |
// normal locking code. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1264 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1265 |
// FIXME: due to a lack of registers we currently blow away the age |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1266 |
// bits in this situation. Should attempt to preserve them. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1267 |
NOT_LP64( movptr(swap_reg, saved_mark_addr); ) |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1268 |
load_prototype_header(tmp_reg, obj_reg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1269 |
if (os::is_MP()) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1270 |
lock(); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1271 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1272 |
cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1273 |
// Fall through to the normal CAS-based lock, because no matter what |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1274 |
// the result of the above CAS, some thread must have succeeded in |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1275 |
// removing the bias bit from the object's header. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1276 |
if (counters != NULL) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1277 |
cond_inc32(Assembler::zero, |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1278 |
ExternalAddress((address) counters->revoked_lock_entry_count_addr())); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1279 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1280 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1281 |
bind(cas_label); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1282 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1283 |
return null_check_offset; |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1284 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1285 |
|
14626 | 1286 |
void MacroAssembler::biased_locking_exit(Register obj_reg, Register temp_reg, Label& done) { |
1287 |
assert(UseBiasedLocking, "why call this otherwise?"); |
|
1288 |
||
1289 |
// Check for biased locking unlock case, which is a no-op |
|
1290 |
// Note: we do not have to check the thread ID for two reasons. |
|
1291 |
// First, the interpreter checks for IllegalMonitorStateException at |
|
1292 |
// a higher level. Second, if the bias was revoked while we held the |
|
1293 |
// lock, the object could not be rebiased toward another thread, so |
|
1294 |
// the bias bit would be clear. |
|
1295 |
movptr(temp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes())); |
|
1296 |
andptr(temp_reg, markOopDesc::biased_lock_mask_in_place); |
|
1297 |
cmpptr(temp_reg, markOopDesc::biased_lock_pattern); |
|
1298 |
jcc(Assembler::equal, done); |
|
1299 |
} |
|
1300 |
||
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1301 |
#ifdef COMPILER2 |
23491 | 1302 |
|
1303 |
#if INCLUDE_RTM_OPT |
|
1304 |
||
1305 |
// Update rtm_counters based on abort status |
|
1306 |
// input: abort_status |
|
1307 |
// rtm_counters (RTMLockingCounters*) |
|
1308 |
// flags are killed |
|
1309 |
void MacroAssembler::rtm_counters_update(Register abort_status, Register rtm_counters) { |
|
1310 |
||
1311 |
atomic_incptr(Address(rtm_counters, RTMLockingCounters::abort_count_offset())); |
|
1312 |
if (PrintPreciseRTMLockingStatistics) { |
|
1313 |
for (int i = 0; i < RTMLockingCounters::ABORT_STATUS_LIMIT; i++) { |
|
1314 |
Label check_abort; |
|
1315 |
testl(abort_status, (1<<i)); |
|
1316 |
jccb(Assembler::equal, check_abort); |
|
1317 |
atomic_incptr(Address(rtm_counters, RTMLockingCounters::abortX_count_offset() + (i * sizeof(uintx)))); |
|
1318 |
bind(check_abort); |
|
1319 |
} |
|
1320 |
} |
|
1321 |
} |
|
1322 |
||
1323 |
// Branch if (random & (count-1) != 0), count is 2^n |
|
1324 |
// tmp, scr and flags are killed |
|
1325 |
void MacroAssembler::branch_on_random_using_rdtsc(Register tmp, Register scr, int count, Label& brLabel) { |
|
1326 |
assert(tmp == rax, ""); |
|
1327 |
assert(scr == rdx, ""); |
|
1328 |
rdtsc(); // modifies EDX:EAX |
|
1329 |
andptr(tmp, count-1); |
|
1330 |
jccb(Assembler::notZero, brLabel); |
|
1331 |
} |
|
1332 |
||
1333 |
// Perform abort ratio calculation, set no_rtm bit if high ratio |
|
1334 |
// input: rtm_counters_Reg (RTMLockingCounters* address) |
|
1335 |
// tmpReg, rtm_counters_Reg and flags are killed |
|
1336 |
void MacroAssembler::rtm_abort_ratio_calculation(Register tmpReg, |
|
1337 |
Register rtm_counters_Reg, |
|
1338 |
RTMLockingCounters* rtm_counters, |
|
1339 |
Metadata* method_data) { |
|
1340 |
Label L_done, L_check_always_rtm1, L_check_always_rtm2; |
|
1341 |
||
1342 |
if (RTMLockingCalculationDelay > 0) { |
|
1343 |
// Delay calculation |
|
1344 |
movptr(tmpReg, ExternalAddress((address) RTMLockingCounters::rtm_calculation_flag_addr()), tmpReg); |
|
1345 |
testptr(tmpReg, tmpReg); |
|
1346 |
jccb(Assembler::equal, L_done); |
|
1347 |
} |
|
1348 |
// Abort ratio calculation only if abort_count > RTMAbortThreshold |
|
1349 |
// Aborted transactions = abort_count * 100 |
|
1350 |
// All transactions = total_count * RTMTotalCountIncrRate |
|
1351 |
// Set no_rtm bit if (Aborted transactions >= All transactions * RTMAbortRatio) |
|
1352 |
||
1353 |
movptr(tmpReg, Address(rtm_counters_Reg, RTMLockingCounters::abort_count_offset())); |
|
1354 |
cmpptr(tmpReg, RTMAbortThreshold); |
|
1355 |
jccb(Assembler::below, L_check_always_rtm2); |
|
1356 |
imulptr(tmpReg, tmpReg, 100); |
|
1357 |
||
1358 |
Register scrReg = rtm_counters_Reg; |
|
1359 |
movptr(scrReg, Address(rtm_counters_Reg, RTMLockingCounters::total_count_offset())); |
|
1360 |
imulptr(scrReg, scrReg, RTMTotalCountIncrRate); |
|
1361 |
imulptr(scrReg, scrReg, RTMAbortRatio); |
|
1362 |
cmpptr(tmpReg, scrReg); |
|
1363 |
jccb(Assembler::below, L_check_always_rtm1); |
|
1364 |
if (method_data != NULL) { |
|
1365 |
// set rtm_state to "no rtm" in MDO |
|
1366 |
mov_metadata(tmpReg, method_data); |
|
1367 |
if (os::is_MP()) { |
|
1368 |
lock(); |
|
1369 |
} |
|
1370 |
orl(Address(tmpReg, MethodData::rtm_state_offset_in_bytes()), NoRTM); |
|
1371 |
} |
|
1372 |
jmpb(L_done); |
|
1373 |
bind(L_check_always_rtm1); |
|
1374 |
// Reload RTMLockingCounters* address |
|
1375 |
lea(rtm_counters_Reg, ExternalAddress((address)rtm_counters)); |
|
1376 |
bind(L_check_always_rtm2); |
|
1377 |
movptr(tmpReg, Address(rtm_counters_Reg, RTMLockingCounters::total_count_offset())); |
|
1378 |
cmpptr(tmpReg, RTMLockingThreshold / RTMTotalCountIncrRate); |
|
1379 |
jccb(Assembler::below, L_done); |
|
1380 |
if (method_data != NULL) { |
|
1381 |
// set rtm_state to "always rtm" in MDO |
|
1382 |
mov_metadata(tmpReg, method_data); |
|
1383 |
if (os::is_MP()) { |
|
1384 |
lock(); |
|
1385 |
} |
|
1386 |
orl(Address(tmpReg, MethodData::rtm_state_offset_in_bytes()), UseRTM); |
|
1387 |
} |
|
1388 |
bind(L_done); |
|
1389 |
} |
|
1390 |
||
1391 |
// Update counters and perform abort ratio calculation |
|
1392 |
// input: abort_status_Reg |
|
1393 |
// rtm_counters_Reg, flags are killed |
|
1394 |
void MacroAssembler::rtm_profiling(Register abort_status_Reg, |
|
1395 |
Register rtm_counters_Reg, |
|
1396 |
RTMLockingCounters* rtm_counters, |
|
1397 |
Metadata* method_data, |
|
1398 |
bool profile_rtm) { |
|
1399 |
||
1400 |
assert(rtm_counters != NULL, "should not be NULL when profiling RTM"); |
|
1401 |
// update rtm counters based on rax value at abort |
|
1402 |
// reads abort_status_Reg, updates flags |
|
1403 |
lea(rtm_counters_Reg, ExternalAddress((address)rtm_counters)); |
|
1404 |
rtm_counters_update(abort_status_Reg, rtm_counters_Reg); |
|
1405 |
if (profile_rtm) { |
|
1406 |
// Save abort status because abort_status_Reg is used by following code. |
|
1407 |
if (RTMRetryCount > 0) { |
|
1408 |
push(abort_status_Reg); |
|
1409 |
} |
|
1410 |
assert(rtm_counters != NULL, "should not be NULL when profiling RTM"); |
|
1411 |
rtm_abort_ratio_calculation(abort_status_Reg, rtm_counters_Reg, rtm_counters, method_data); |
|
1412 |
// restore abort status |
|
1413 |
if (RTMRetryCount > 0) { |
|
1414 |
pop(abort_status_Reg); |
|
1415 |
} |
|
1416 |
} |
|
1417 |
} |
|
1418 |
||
1419 |
// Retry on abort if abort's status is 0x6: can retry (0x2) | memory conflict (0x4) |
|
1420 |
// inputs: retry_count_Reg |
|
1421 |
// : abort_status_Reg |
|
1422 |
// output: retry_count_Reg decremented by 1 |
|
1423 |
// flags are killed |
|
1424 |
void MacroAssembler::rtm_retry_lock_on_abort(Register retry_count_Reg, Register abort_status_Reg, Label& retryLabel) { |
|
1425 |
Label doneRetry; |
|
1426 |
assert(abort_status_Reg == rax, ""); |
|
1427 |
// The abort reason bits are in eax (see all states in rtmLocking.hpp) |
|
1428 |
// 0x6 = conflict on which we can retry (0x2) | memory conflict (0x4) |
|
1429 |
// if reason is in 0x6 and retry count != 0 then retry |
|
1430 |
andptr(abort_status_Reg, 0x6); |
|
1431 |
jccb(Assembler::zero, doneRetry); |
|
1432 |
testl(retry_count_Reg, retry_count_Reg); |
|
1433 |
jccb(Assembler::zero, doneRetry); |
|
1434 |
pause(); |
|
1435 |
decrementl(retry_count_Reg); |
|
1436 |
jmp(retryLabel); |
|
1437 |
bind(doneRetry); |
|
1438 |
} |
|
1439 |
||
1440 |
// Spin and retry if lock is busy, |
|
1441 |
// inputs: box_Reg (monitor address) |
|
1442 |
// : retry_count_Reg |
|
1443 |
// output: retry_count_Reg decremented by 1 |
|
1444 |
// : clear z flag if retry count exceeded |
|
1445 |
// tmp_Reg, scr_Reg, flags are killed |
|
1446 |
void MacroAssembler::rtm_retry_lock_on_busy(Register retry_count_Reg, Register box_Reg, |
|
1447 |
Register tmp_Reg, Register scr_Reg, Label& retryLabel) { |
|
1448 |
Label SpinLoop, SpinExit, doneRetry; |
|
27608 | 1449 |
int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner); |
23491 | 1450 |
|
1451 |
testl(retry_count_Reg, retry_count_Reg); |
|
1452 |
jccb(Assembler::zero, doneRetry); |
|
1453 |
decrementl(retry_count_Reg); |
|
1454 |
movptr(scr_Reg, RTMSpinLoopCount); |
|
1455 |
||
1456 |
bind(SpinLoop); |
|
1457 |
pause(); |
|
1458 |
decrementl(scr_Reg); |
|
1459 |
jccb(Assembler::lessEqual, SpinExit); |
|
1460 |
movptr(tmp_Reg, Address(box_Reg, owner_offset)); |
|
1461 |
testptr(tmp_Reg, tmp_Reg); |
|
1462 |
jccb(Assembler::notZero, SpinLoop); |
|
1463 |
||
1464 |
bind(SpinExit); |
|
1465 |
jmp(retryLabel); |
|
1466 |
bind(doneRetry); |
|
1467 |
incrementl(retry_count_Reg); // clear z flag |
|
1468 |
} |
|
1469 |
||
1470 |
// Use RTM for normal stack locks |
|
1471 |
// Input: objReg (object to lock) |
|
1472 |
void MacroAssembler::rtm_stack_locking(Register objReg, Register tmpReg, Register scrReg, |
|
1473 |
Register retry_on_abort_count_Reg, |
|
1474 |
RTMLockingCounters* stack_rtm_counters, |
|
1475 |
Metadata* method_data, bool profile_rtm, |
|
1476 |
Label& DONE_LABEL, Label& IsInflated) { |
|
1477 |
assert(UseRTMForStackLocks, "why call this otherwise?"); |
|
1478 |
assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking"); |
|
1479 |
assert(tmpReg == rax, ""); |
|
1480 |
assert(scrReg == rdx, ""); |
|
1481 |
Label L_rtm_retry, L_decrement_retry, L_on_abort; |
|
1482 |
||
1483 |
if (RTMRetryCount > 0) { |
|
1484 |
movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort |
|
1485 |
bind(L_rtm_retry); |
|
1486 |
} |
|
23847
d792e42aeb4f
8038939: Some options related to RTM locking optimization works inconsistently
kvn
parents:
23491
diff
changeset
|
1487 |
movptr(tmpReg, Address(objReg, 0)); |
d792e42aeb4f
8038939: Some options related to RTM locking optimization works inconsistently
kvn
parents:
23491
diff
changeset
|
1488 |
testptr(tmpReg, markOopDesc::monitor_value); // inflated vs stack-locked|neutral|biased |
d792e42aeb4f
8038939: Some options related to RTM locking optimization works inconsistently
kvn
parents:
23491
diff
changeset
|
1489 |
jcc(Assembler::notZero, IsInflated); |
d792e42aeb4f
8038939: Some options related to RTM locking optimization works inconsistently
kvn
parents:
23491
diff
changeset
|
1490 |
|
23491 | 1491 |
if (PrintPreciseRTMLockingStatistics || profile_rtm) { |
1492 |
Label L_noincrement; |
|
1493 |
if (RTMTotalCountIncrRate > 1) { |
|
1494 |
// tmpReg, scrReg and flags are killed |
|
1495 |
branch_on_random_using_rdtsc(tmpReg, scrReg, (int)RTMTotalCountIncrRate, L_noincrement); |
|
1496 |
} |
|
1497 |
assert(stack_rtm_counters != NULL, "should not be NULL when profiling RTM"); |
|
1498 |
atomic_incptr(ExternalAddress((address)stack_rtm_counters->total_count_addr()), scrReg); |
|
1499 |
bind(L_noincrement); |
|
1500 |
} |
|
1501 |
xbegin(L_on_abort); |
|
1502 |
movptr(tmpReg, Address(objReg, 0)); // fetch markword |
|
1503 |
andptr(tmpReg, markOopDesc::biased_lock_mask_in_place); // look at 3 lock bits |
|
1504 |
cmpptr(tmpReg, markOopDesc::unlocked_value); // bits = 001 unlocked |
|
1505 |
jcc(Assembler::equal, DONE_LABEL); // all done if unlocked |
|
1506 |
||
1507 |
Register abort_status_Reg = tmpReg; // status of abort is stored in RAX |
|
1508 |
if (UseRTMXendForLockBusy) { |
|
1509 |
xend(); |
|
23847
d792e42aeb4f
8038939: Some options related to RTM locking optimization works inconsistently
kvn
parents:
23491
diff
changeset
|
1510 |
movptr(abort_status_Reg, 0x2); // Set the abort status to 2 (so we can retry) |
23491 | 1511 |
jmp(L_decrement_retry); |
1512 |
} |
|
1513 |
else { |
|
1514 |
xabort(0); |
|
1515 |
} |
|
1516 |
bind(L_on_abort); |
|
1517 |
if (PrintPreciseRTMLockingStatistics || profile_rtm) { |
|
1518 |
rtm_profiling(abort_status_Reg, scrReg, stack_rtm_counters, method_data, profile_rtm); |
|
1519 |
} |
|
1520 |
bind(L_decrement_retry); |
|
1521 |
if (RTMRetryCount > 0) { |
|
1522 |
// retry on lock abort if abort status is 'can retry' (0x2) or 'memory conflict' (0x4) |
|
1523 |
rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry); |
|
1524 |
} |
|
1525 |
} |
|
1526 |
||
1527 |
// Use RTM for inflating locks |
|
1528 |
// inputs: objReg (object to lock) |
|
1529 |
// boxReg (on-stack box address (displaced header location) - KILLED) |
|
27608 | 1530 |
// tmpReg (ObjectMonitor address + markOopDesc::monitor_value) |
23491 | 1531 |
void MacroAssembler::rtm_inflated_locking(Register objReg, Register boxReg, Register tmpReg, |
1532 |
Register scrReg, Register retry_on_busy_count_Reg, |
|
1533 |
Register retry_on_abort_count_Reg, |
|
1534 |
RTMLockingCounters* rtm_counters, |
|
1535 |
Metadata* method_data, bool profile_rtm, |
|
1536 |
Label& DONE_LABEL) { |
|
1537 |
assert(UseRTMLocking, "why call this otherwise?"); |
|
1538 |
assert(tmpReg == rax, ""); |
|
1539 |
assert(scrReg == rdx, ""); |
|
1540 |
Label L_rtm_retry, L_decrement_retry, L_on_abort; |
|
27608 | 1541 |
int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner); |
23491 | 1542 |
|
1543 |
// Without cast to int32_t a movptr will destroy r10 which is typically obj |
|
1544 |
movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark())); |
|
1545 |
movptr(boxReg, tmpReg); // Save ObjectMonitor address |
|
1546 |
||
1547 |
if (RTMRetryCount > 0) { |
|
1548 |
movl(retry_on_busy_count_Reg, RTMRetryCount); // Retry on lock busy |
|
1549 |
movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort |
|
1550 |
bind(L_rtm_retry); |
|
1551 |
} |
|
1552 |
if (PrintPreciseRTMLockingStatistics || profile_rtm) { |
|
1553 |
Label L_noincrement; |
|
1554 |
if (RTMTotalCountIncrRate > 1) { |
|
1555 |
// tmpReg, scrReg and flags are killed |
|
1556 |
branch_on_random_using_rdtsc(tmpReg, scrReg, (int)RTMTotalCountIncrRate, L_noincrement); |
|
1557 |
} |
|
1558 |
assert(rtm_counters != NULL, "should not be NULL when profiling RTM"); |
|
1559 |
atomic_incptr(ExternalAddress((address)rtm_counters->total_count_addr()), scrReg); |
|
1560 |
bind(L_noincrement); |
|
1561 |
} |
|
1562 |
xbegin(L_on_abort); |
|
1563 |
movptr(tmpReg, Address(objReg, 0)); |
|
1564 |
movptr(tmpReg, Address(tmpReg, owner_offset)); |
|
1565 |
testptr(tmpReg, tmpReg); |
|
1566 |
jcc(Assembler::zero, DONE_LABEL); |
|
1567 |
if (UseRTMXendForLockBusy) { |
|
1568 |
xend(); |
|
1569 |
jmp(L_decrement_retry); |
|
1570 |
} |
|
1571 |
else { |
|
1572 |
xabort(0); |
|
1573 |
} |
|
1574 |
bind(L_on_abort); |
|
1575 |
Register abort_status_Reg = tmpReg; // status of abort is stored in RAX |
|
1576 |
if (PrintPreciseRTMLockingStatistics || profile_rtm) { |
|
1577 |
rtm_profiling(abort_status_Reg, scrReg, rtm_counters, method_data, profile_rtm); |
|
1578 |
} |
|
1579 |
if (RTMRetryCount > 0) { |
|
1580 |
// retry on lock abort if abort status is 'can retry' (0x2) or 'memory conflict' (0x4) |
|
1581 |
rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry); |
|
1582 |
} |
|
1583 |
||
1584 |
movptr(tmpReg, Address(boxReg, owner_offset)) ; |
|
1585 |
testptr(tmpReg, tmpReg) ; |
|
1586 |
jccb(Assembler::notZero, L_decrement_retry) ; |
|
1587 |
||
1588 |
// Appears unlocked - try to swing _owner from null to non-null. |
|
1589 |
// Invariant: tmpReg == 0. tmpReg is EAX which is the implicit cmpxchg comparand. |
|
1590 |
#ifdef _LP64 |
|
1591 |
Register threadReg = r15_thread; |
|
1592 |
#else |
|
1593 |
get_thread(scrReg); |
|
1594 |
Register threadReg = scrReg; |
|
1595 |
#endif |
|
1596 |
if (os::is_MP()) { |
|
1597 |
lock(); |
|
1598 |
} |
|
1599 |
cmpxchgptr(threadReg, Address(boxReg, owner_offset)); // Updates tmpReg |
|
1600 |
||
1601 |
if (RTMRetryCount > 0) { |
|
1602 |
// success done else retry |
|
1603 |
jccb(Assembler::equal, DONE_LABEL) ; |
|
1604 |
bind(L_decrement_retry); |
|
1605 |
// Spin and retry if lock is busy. |
|
1606 |
rtm_retry_lock_on_busy(retry_on_busy_count_Reg, boxReg, tmpReg, scrReg, L_rtm_retry); |
|
1607 |
} |
|
1608 |
else { |
|
1609 |
bind(L_decrement_retry); |
|
1610 |
} |
|
1611 |
} |
|
1612 |
||
1613 |
#endif // INCLUDE_RTM_OPT |
|
1614 |
||
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1615 |
// Fast_Lock and Fast_Unlock used by C2 |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1616 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1617 |
// Because the transitions from emitted code to the runtime |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1618 |
// monitorenter/exit helper stubs are so slow it's critical that |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1619 |
// we inline both the stack-locking fast-path and the inflated fast path. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1620 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1621 |
// See also: cmpFastLock and cmpFastUnlock. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1622 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1623 |
// What follows is a specialized inline transliteration of the code |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1624 |
// in slow_enter() and slow_exit(). If we're concerned about I$ bloat |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1625 |
// another option would be to emit TrySlowEnter and TrySlowExit methods |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1626 |
// at startup-time. These methods would accept arguments as |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1627 |
// (rax,=Obj, rbx=Self, rcx=box, rdx=Scratch) and return success-failure |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1628 |
// indications in the icc.ZFlag. Fast_Lock and Fast_Unlock would simply |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1629 |
// marshal the arguments and emit calls to TrySlowEnter and TrySlowExit. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1630 |
// In practice, however, the # of lock sites is bounded and is usually small. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1631 |
// Besides the call overhead, TrySlowEnter and TrySlowExit might suffer |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1632 |
// if the processor uses simple bimodal branch predictors keyed by EIP |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1633 |
// Since the helper routines would be called from multiple synchronization |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1634 |
// sites. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1635 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1636 |
// An even better approach would be write "MonitorEnter()" and "MonitorExit()" |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1637 |
// in java - using j.u.c and unsafe - and just bind the lock and unlock sites |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1638 |
// to those specialized methods. That'd give us a mostly platform-independent |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1639 |
// implementation that the JITs could optimize and inline at their pleasure. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1640 |
// Done correctly, the only time we'd need to cross to native could would be |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1641 |
// to park() or unpark() threads. We'd also need a few more unsafe operators |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1642 |
// to (a) prevent compiler-JIT reordering of non-volatile accesses, and |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1643 |
// (b) explicit barriers or fence operations. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1644 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1645 |
// TODO: |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1646 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1647 |
// * Arrange for C2 to pass "Self" into Fast_Lock and Fast_Unlock in one of the registers (scr). |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1648 |
// This avoids manifesting the Self pointer in the Fast_Lock and Fast_Unlock terminals. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1649 |
// Given TLAB allocation, Self is usually manifested in a register, so passing it into |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1650 |
// the lock operators would typically be faster than reifying Self. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1651 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1652 |
// * Ideally I'd define the primitives as: |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1653 |
// fast_lock (nax Obj, nax box, EAX tmp, nax scr) where box, tmp and scr are KILLED. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1654 |
// fast_unlock (nax Obj, EAX box, nax tmp) where box and tmp are KILLED |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1655 |
// Unfortunately ADLC bugs prevent us from expressing the ideal form. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1656 |
// Instead, we're stuck with a rather awkward and brittle register assignments below. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1657 |
// Furthermore the register assignments are overconstrained, possibly resulting in |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1658 |
// sub-optimal code near the synchronization site. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1659 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1660 |
// * Eliminate the sp-proximity tests and just use "== Self" tests instead. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1661 |
// Alternately, use a better sp-proximity test. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1662 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1663 |
// * Currently ObjectMonitor._Owner can hold either an sp value or a (THREAD *) value. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1664 |
// Either one is sufficient to uniquely identify a thread. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1665 |
// TODO: eliminate use of sp in _owner and use get_thread(tr) instead. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1666 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1667 |
// * Intrinsify notify() and notifyAll() for the common cases where the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1668 |
// object is locked by the calling thread but the waitlist is empty. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1669 |
// avoid the expensive JNI call to JVM_Notify() and JVM_NotifyAll(). |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1670 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1671 |
// * use jccb and jmpb instead of jcc and jmp to improve code density. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1672 |
// But beware of excessive branch density on AMD Opterons. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1673 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1674 |
// * Both Fast_Lock and Fast_Unlock set the ICC.ZF to indicate success |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1675 |
// or failure of the fast-path. If the fast-path fails then we pass |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1676 |
// control to the slow-path, typically in C. In Fast_Lock and |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1677 |
// Fast_Unlock we often branch to DONE_LABEL, just to find that C2 |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1678 |
// will emit a conditional branch immediately after the node. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1679 |
// So we have branches to branches and lots of ICC.ZF games. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1680 |
// Instead, it might be better to have C2 pass a "FailureLabel" |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1681 |
// into Fast_Lock and Fast_Unlock. In the case of success, control |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1682 |
// will drop through the node. ICC.ZF is undefined at exit. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1683 |
// In the case of failure, the node will branch directly to the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1684 |
// FailureLabel |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1685 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1686 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1687 |
// obj: object to lock |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1688 |
// box: on-stack box address (displaced header location) - KILLED |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1689 |
// rax,: tmp -- KILLED |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1690 |
// scr: tmp -- KILLED |
23491 | 1691 |
void MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmpReg, |
1692 |
Register scrReg, Register cx1Reg, Register cx2Reg, |
|
1693 |
BiasedLockingCounters* counters, |
|
1694 |
RTMLockingCounters* rtm_counters, |
|
1695 |
RTMLockingCounters* stack_rtm_counters, |
|
1696 |
Metadata* method_data, |
|
1697 |
bool use_rtm, bool profile_rtm) { |
|
37251
9fc139ad74b5
8152358: code and comment cleanups found during the hunt for 8077392
dcubed
parents:
36561
diff
changeset
|
1698 |
// Ensure the register assignments are disjoint |
23491 | 1699 |
assert(tmpReg == rax, ""); |
1700 |
||
1701 |
if (use_rtm) { |
|
1702 |
assert_different_registers(objReg, boxReg, tmpReg, scrReg, cx1Reg, cx2Reg); |
|
1703 |
} else { |
|
1704 |
assert(cx1Reg == noreg, ""); |
|
1705 |
assert(cx2Reg == noreg, ""); |
|
1706 |
assert_different_registers(objReg, boxReg, tmpReg, scrReg); |
|
1707 |
} |
|
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1708 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1709 |
if (counters != NULL) { |
23491 | 1710 |
atomic_incl(ExternalAddress((address)counters->total_entry_count_addr()), scrReg); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1711 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1712 |
if (EmitSync & 1) { |
27608 | 1713 |
// set box->dhw = markOopDesc::unused_mark() |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1714 |
// Force all sync thru slow-path: slow_enter() and slow_exit() |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1715 |
movptr (Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark())); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1716 |
cmpptr (rsp, (int32_t)NULL_WORD); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1717 |
} else { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1718 |
// Possible cases that we'll encounter in fast_lock |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1719 |
// ------------------------------------------------ |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1720 |
// * Inflated |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1721 |
// -- unlocked |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1722 |
// -- Locked |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1723 |
// = by self |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1724 |
// = by other |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1725 |
// * biased |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1726 |
// -- by Self |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1727 |
// -- by other |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1728 |
// * neutral |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1729 |
// * stack-locked |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1730 |
// -- by self |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1731 |
// = sp-proximity test hits |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1732 |
// = sp-proximity test generates false-negative |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1733 |
// -- by other |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1734 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1735 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1736 |
Label IsInflated, DONE_LABEL; |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1737 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1738 |
// it's stack-locked, biased or neutral |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1739 |
// TODO: optimize away redundant LDs of obj->mark and improve the markword triage |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1740 |
// order to reduce the number of conditional branches in the most common cases. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1741 |
// Beware -- there's a subtle invariant that fetch of the markword |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1742 |
// at [FETCH], below, will never observe a biased encoding (*101b). |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1743 |
// If this invariant is not held we risk exclusion (safety) failure. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1744 |
if (UseBiasedLocking && !UseOptoBiasInlining) { |
27638
1142a24d73eb
8062950: Bug in locking code when UseOptoBiasInlining is disabled: assert(dmw->is_neutral()) failed: invariant
mdoerr
parents:
26434
diff
changeset
|
1745 |
biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, counters); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1746 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1747 |
|
23491 | 1748 |
#if INCLUDE_RTM_OPT |
1749 |
if (UseRTMForStackLocks && use_rtm) { |
|
1750 |
rtm_stack_locking(objReg, tmpReg, scrReg, cx2Reg, |
|
1751 |
stack_rtm_counters, method_data, profile_rtm, |
|
1752 |
DONE_LABEL, IsInflated); |
|
1753 |
} |
|
1754 |
#endif // INCLUDE_RTM_OPT |
|
1755 |
||
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1756 |
movptr(tmpReg, Address(objReg, 0)); // [FETCH] |
23491 | 1757 |
testptr(tmpReg, markOopDesc::monitor_value); // inflated vs stack-locked|neutral|biased |
1758 |
jccb(Assembler::notZero, IsInflated); |
|
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1759 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1760 |
// Attempt stack-locking ... |
23491 | 1761 |
orptr (tmpReg, markOopDesc::unlocked_value); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1762 |
movptr(Address(boxReg, 0), tmpReg); // Anticipate successful CAS |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1763 |
if (os::is_MP()) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1764 |
lock(); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1765 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1766 |
cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1767 |
if (counters != NULL) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1768 |
cond_inc32(Assembler::equal, |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1769 |
ExternalAddress((address)counters->fast_path_entry_count_addr())); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1770 |
} |
23491 | 1771 |
jcc(Assembler::equal, DONE_LABEL); // Success |
1772 |
||
1773 |
// Recursive locking. |
|
1774 |
// The object is stack-locked: markword contains stack pointer to BasicLock. |
|
1775 |
// Locked by current thread if difference with current SP is less than one page. |
|
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1776 |
subptr(tmpReg, rsp); |
23491 | 1777 |
// Next instruction set ZFlag == 1 (Success) if difference is less then one page. |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1778 |
andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - os::vm_page_size())) ); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1779 |
movptr(Address(boxReg, 0), tmpReg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1780 |
if (counters != NULL) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1781 |
cond_inc32(Assembler::equal, |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1782 |
ExternalAddress((address)counters->fast_path_entry_count_addr())); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1783 |
} |
23491 | 1784 |
jmp(DONE_LABEL); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1785 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1786 |
bind(IsInflated); |
27608 | 1787 |
// The object is inflated. tmpReg contains pointer to ObjectMonitor* + markOopDesc::monitor_value |
23491 | 1788 |
|
1789 |
#if INCLUDE_RTM_OPT |
|
1790 |
// Use the same RTM locking code in 32- and 64-bit VM. |
|
1791 |
if (use_rtm) { |
|
1792 |
rtm_inflated_locking(objReg, boxReg, tmpReg, scrReg, cx1Reg, cx2Reg, |
|
1793 |
rtm_counters, method_data, profile_rtm, DONE_LABEL); |
|
1794 |
} else { |
|
1795 |
#endif // INCLUDE_RTM_OPT |
|
1796 |
||
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1797 |
#ifndef _LP64 |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1798 |
// The object is inflated. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1799 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1800 |
// boxReg refers to the on-stack BasicLock in the current frame. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1801 |
// We'd like to write: |
27608 | 1802 |
// set box->_displaced_header = markOopDesc::unused_mark(). Any non-0 value suffices. |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1803 |
// This is convenient but results a ST-before-CAS penalty. The following CAS suffers |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1804 |
// additional latency as we have another ST in the store buffer that must drain. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1805 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1806 |
if (EmitSync & 8192) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1807 |
movptr(Address(boxReg, 0), 3); // results in ST-before-CAS penalty |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1808 |
get_thread (scrReg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1809 |
movptr(boxReg, tmpReg); // consider: LEA box, [tmp-2] |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1810 |
movptr(tmpReg, NULL_WORD); // consider: xor vs mov |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1811 |
if (os::is_MP()) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1812 |
lock(); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1813 |
} |
27608 | 1814 |
cmpxchgptr(scrReg, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1815 |
} else |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1816 |
if ((EmitSync & 128) == 0) { // avoid ST-before-CAS |
31782
b23b74f8ae8d
8130448: thread dump improvements, comment additions, new diagnostics inspired by 8077392
dcubed
parents:
31592
diff
changeset
|
1817 |
// register juggle because we need tmpReg for cmpxchgptr below |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1818 |
movptr(scrReg, boxReg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1819 |
movptr(boxReg, tmpReg); // consider: LEA box, [tmp-2] |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1820 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1821 |
// Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1822 |
if ((EmitSync & 2048) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1823 |
// prefetchw [eax + Offset(_owner)-2] |
27608 | 1824 |
prefetchw(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1825 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1826 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1827 |
if ((EmitSync & 64) == 0) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1828 |
// Optimistic form: consider XORL tmpReg,tmpReg |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1829 |
movptr(tmpReg, NULL_WORD); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1830 |
} else { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1831 |
// Can suffer RTS->RTO upgrades on shared or cold $ lines |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1832 |
// Test-And-CAS instead of CAS |
27608 | 1833 |
movptr(tmpReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); // rax, = m->_owner |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1834 |
testptr(tmpReg, tmpReg); // Locked ? |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1835 |
jccb (Assembler::notZero, DONE_LABEL); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1836 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1837 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1838 |
// Appears unlocked - try to swing _owner from null to non-null. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1839 |
// Ideally, I'd manifest "Self" with get_thread and then attempt |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1840 |
// to CAS the register containing Self into m->Owner. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1841 |
// But we don't have enough registers, so instead we can either try to CAS |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1842 |
// rsp or the address of the box (in scr) into &m->owner. If the CAS succeeds |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1843 |
// we later store "Self" into m->Owner. Transiently storing a stack address |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1844 |
// (rsp or the address of the box) into m->owner is harmless. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1845 |
// Invariant: tmpReg == 0. tmpReg is EAX which is the implicit cmpxchg comparand. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1846 |
if (os::is_MP()) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1847 |
lock(); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1848 |
} |
27608 | 1849 |
cmpxchgptr(scrReg, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1850 |
movptr(Address(scrReg, 0), 3); // box->_displaced_header = 3 |
31782
b23b74f8ae8d
8130448: thread dump improvements, comment additions, new diagnostics inspired by 8077392
dcubed
parents:
31592
diff
changeset
|
1851 |
// If we weren't able to swing _owner from NULL to the BasicLock |
b23b74f8ae8d
8130448: thread dump improvements, comment additions, new diagnostics inspired by 8077392
dcubed
parents:
31592
diff
changeset
|
1852 |
// then take the slow path. |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1853 |
jccb (Assembler::notZero, DONE_LABEL); |
31782
b23b74f8ae8d
8130448: thread dump improvements, comment additions, new diagnostics inspired by 8077392
dcubed
parents:
31592
diff
changeset
|
1854 |
// update _owner from BasicLock to thread |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1855 |
get_thread (scrReg); // beware: clobbers ICCs |
27608 | 1856 |
movptr(Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), scrReg); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1857 |
xorptr(boxReg, boxReg); // set icc.ZFlag = 1 to indicate success |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1858 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1859 |
// If the CAS fails we can either retry or pass control to the slow-path. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1860 |
// We use the latter tactic. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1861 |
// Pass the CAS result in the icc.ZFlag into DONE_LABEL |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1862 |
// If the CAS was successful ... |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1863 |
// Self has acquired the lock |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1864 |
// Invariant: m->_recursions should already be 0, so we don't need to explicitly set it. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1865 |
// Intentional fall-through into DONE_LABEL ... |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1866 |
} else { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1867 |
movptr(Address(boxReg, 0), intptr_t(markOopDesc::unused_mark())); // results in ST-before-CAS penalty |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1868 |
movptr(boxReg, tmpReg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1869 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1870 |
// Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1871 |
if ((EmitSync & 2048) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1872 |
// prefetchw [eax + Offset(_owner)-2] |
27608 | 1873 |
prefetchw(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1874 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1875 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1876 |
if ((EmitSync & 64) == 0) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1877 |
// Optimistic form |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1878 |
xorptr (tmpReg, tmpReg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1879 |
} else { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1880 |
// Can suffer RTS->RTO upgrades on shared or cold $ lines |
27608 | 1881 |
movptr(tmpReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); // rax, = m->_owner |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1882 |
testptr(tmpReg, tmpReg); // Locked ? |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1883 |
jccb (Assembler::notZero, DONE_LABEL); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1884 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1885 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1886 |
// Appears unlocked - try to swing _owner from null to non-null. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1887 |
// Use either "Self" (in scr) or rsp as thread identity in _owner. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1888 |
// Invariant: tmpReg == 0. tmpReg is EAX which is the implicit cmpxchg comparand. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1889 |
get_thread (scrReg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1890 |
if (os::is_MP()) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1891 |
lock(); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1892 |
} |
27608 | 1893 |
cmpxchgptr(scrReg, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1894 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1895 |
// If the CAS fails we can either retry or pass control to the slow-path. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1896 |
// We use the latter tactic. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1897 |
// Pass the CAS result in the icc.ZFlag into DONE_LABEL |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1898 |
// If the CAS was successful ... |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1899 |
// Self has acquired the lock |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1900 |
// Invariant: m->_recursions should already be 0, so we don't need to explicitly set it. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1901 |
// Intentional fall-through into DONE_LABEL ... |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1902 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1903 |
#else // _LP64 |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1904 |
// It's inflated |
29070 | 1905 |
movq(scrReg, tmpReg); |
1906 |
xorq(tmpReg, tmpReg); |
|
1907 |
||
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1908 |
if (os::is_MP()) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1909 |
lock(); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1910 |
} |
29070 | 1911 |
cmpxchgptr(r15_thread, Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); |
1912 |
// Unconditionally set box->_displaced_header = markOopDesc::unused_mark(). |
|
1913 |
// Without cast to int32_t movptr will destroy r10 which is typically obj. |
|
1914 |
movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark())); |
|
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1915 |
// Intentional fall-through into DONE_LABEL ... |
29070 | 1916 |
// Propagate ICC.ZF from CAS above into DONE_LABEL. |
23491 | 1917 |
#endif // _LP64 |
1918 |
#if INCLUDE_RTM_OPT |
|
1919 |
} // use_rtm() |
|
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1920 |
#endif |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1921 |
// DONE_LABEL is a hot target - we'd really like to place it at the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1922 |
// start of cache line by padding with NOPs. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1923 |
// See the AMD and Intel software optimization manuals for the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1924 |
// most efficient "long" NOP encodings. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1925 |
// Unfortunately none of our alignment mechanisms suffice. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1926 |
bind(DONE_LABEL); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1927 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1928 |
// At DONE_LABEL the icc ZFlag is set as follows ... |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1929 |
// Fast_Unlock uses the same protocol. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1930 |
// ZFlag == 1 -> Success |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1931 |
// ZFlag == 0 -> Failure - force control through the slow-path |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1932 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1933 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1934 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1935 |
// obj: object to unlock |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1936 |
// box: box address (displaced header location), killed. Must be EAX. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1937 |
// tmp: killed, cannot be obj nor box. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1938 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1939 |
// Some commentary on balanced locking: |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1940 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1941 |
// Fast_Lock and Fast_Unlock are emitted only for provably balanced lock sites. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1942 |
// Methods that don't have provably balanced locking are forced to run in the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1943 |
// interpreter - such methods won't be compiled to use fast_lock and fast_unlock. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1944 |
// The interpreter provides two properties: |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1945 |
// I1: At return-time the interpreter automatically and quietly unlocks any |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1946 |
// objects acquired the current activation (frame). Recall that the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1947 |
// interpreter maintains an on-stack list of locks currently held by |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1948 |
// a frame. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1949 |
// I2: If a method attempts to unlock an object that is not held by the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1950 |
// the frame the interpreter throws IMSX. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1951 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1952 |
// Lets say A(), which has provably balanced locking, acquires O and then calls B(). |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1953 |
// B() doesn't have provably balanced locking so it runs in the interpreter. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1954 |
// Control returns to A() and A() unlocks O. By I1 and I2, above, we know that O |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1955 |
// is still locked by A(). |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1956 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1957 |
// The only other source of unbalanced locking would be JNI. The "Java Native Interface: |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1958 |
// Programmer's Guide and Specification" claims that an object locked by jni_monitorenter |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1959 |
// should not be unlocked by "normal" java-level locking and vice-versa. The specification |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1960 |
// doesn't specify what will occur if a program engages in such mixed-mode locking, however. |
30244 | 1961 |
// Arguably given that the spec legislates the JNI case as undefined our implementation |
1962 |
// could reasonably *avoid* checking owner in Fast_Unlock(). |
|
1963 |
// In the interest of performance we elide m->Owner==Self check in unlock. |
|
1964 |
// A perfectly viable alternative is to elide the owner check except when |
|
1965 |
// Xcheck:jni is enabled. |
|
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1966 |
|
23491 | 1967 |
void MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register tmpReg, bool use_rtm) { |
1968 |
assert(boxReg == rax, ""); |
|
1969 |
assert_different_registers(objReg, boxReg, tmpReg); |
|
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1970 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1971 |
if (EmitSync & 4) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1972 |
// Disable - inhibit all inlining. Force control through the slow-path |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1973 |
cmpptr (rsp, 0); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1974 |
} else { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1975 |
Label DONE_LABEL, Stacked, CheckSucc; |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1976 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1977 |
// Critically, the biased locking test must have precedence over |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1978 |
// and appear before the (box->dhw == 0) recursive stack-lock test. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1979 |
if (UseBiasedLocking && !UseOptoBiasInlining) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1980 |
biased_locking_exit(objReg, tmpReg, DONE_LABEL); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1981 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1982 |
|
23491 | 1983 |
#if INCLUDE_RTM_OPT |
1984 |
if (UseRTMForStackLocks && use_rtm) { |
|
1985 |
assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking"); |
|
1986 |
Label L_regular_unlock; |
|
1987 |
movptr(tmpReg, Address(objReg, 0)); // fetch markword |
|
1988 |
andptr(tmpReg, markOopDesc::biased_lock_mask_in_place); // look at 3 lock bits |
|
1989 |
cmpptr(tmpReg, markOopDesc::unlocked_value); // bits = 001 unlocked |
|
1990 |
jccb(Assembler::notEqual, L_regular_unlock); // if !HLE RegularLock |
|
1991 |
xend(); // otherwise end... |
|
1992 |
jmp(DONE_LABEL); // ... and we're done |
|
1993 |
bind(L_regular_unlock); |
|
1994 |
} |
|
1995 |
#endif |
|
1996 |
||
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1997 |
cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD); // Examine the displaced header |
23491 | 1998 |
jcc (Assembler::zero, DONE_LABEL); // 0 indicates recursive stack-lock |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1999 |
movptr(tmpReg, Address(objReg, 0)); // Examine the object's markword |
23491 | 2000 |
testptr(tmpReg, markOopDesc::monitor_value); // Inflated? |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2001 |
jccb (Assembler::zero, Stacked); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2002 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2003 |
// It's inflated. |
23491 | 2004 |
#if INCLUDE_RTM_OPT |
2005 |
if (use_rtm) { |
|
2006 |
Label L_regular_inflated_unlock; |
|
27608 | 2007 |
int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner); |
23491 | 2008 |
movptr(boxReg, Address(tmpReg, owner_offset)); |
2009 |
testptr(boxReg, boxReg); |
|
2010 |
jccb(Assembler::notZero, L_regular_inflated_unlock); |
|
2011 |
xend(); |
|
2012 |
jmpb(DONE_LABEL); |
|
2013 |
bind(L_regular_inflated_unlock); |
|
2014 |
} |
|
2015 |
#endif |
|
2016 |
||
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2017 |
// Despite our balanced locking property we still check that m->_owner == Self |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2018 |
// as java routines or native JNI code called by this thread might |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2019 |
// have released the lock. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2020 |
// Refer to the comments in synchronizer.cpp for how we might encode extra |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2021 |
// state in _succ so we can avoid fetching EntryList|cxq. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2022 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2023 |
// I'd like to add more cases in fast_lock() and fast_unlock() -- |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2024 |
// such as recursive enter and exit -- but we have to be wary of |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2025 |
// I$ bloat, T$ effects and BP$ effects. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2026 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2027 |
// If there's no contention try a 1-0 exit. That is, exit without |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2028 |
// a costly MEMBAR or CAS. See synchronizer.cpp for details on how |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2029 |
// we detect and recover from the race that the 1-0 exit admits. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2030 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2031 |
// Conceptually Fast_Unlock() must execute a STST|LDST "release" barrier |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2032 |
// before it STs null into _owner, releasing the lock. Updates |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2033 |
// to data protected by the critical section must be visible before |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2034 |
// we drop the lock (and thus before any other thread could acquire |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2035 |
// the lock and observe the fields protected by the lock). |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2036 |
// IA32's memory-model is SPO, so STs are ordered with respect to |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2037 |
// each other and there's no need for an explicit barrier (fence). |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2038 |
// See also http://gee.cs.oswego.edu/dl/jmm/cookbook.html. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2039 |
#ifndef _LP64 |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2040 |
get_thread (boxReg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2041 |
if ((EmitSync & 4096) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2042 |
// prefetchw [ebx + Offset(_owner)-2] |
27608 | 2043 |
prefetchw(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2044 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2045 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2046 |
// Note that we could employ various encoding schemes to reduce |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2047 |
// the number of loads below (currently 4) to just 2 or 3. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2048 |
// Refer to the comments in synchronizer.cpp. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2049 |
// In practice the chain of fetches doesn't seem to impact performance, however. |
30244 | 2050 |
xorptr(boxReg, boxReg); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2051 |
if ((EmitSync & 65536) == 0 && (EmitSync & 256)) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2052 |
// Attempt to reduce branch density - AMD's branch predictor. |
27608 | 2053 |
orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions))); |
2054 |
orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList))); |
|
2055 |
orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq))); |
|
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2056 |
jccb (Assembler::notZero, DONE_LABEL); |
27608 | 2057 |
movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2058 |
jmpb (DONE_LABEL); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2059 |
} else { |
27608 | 2060 |
orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions))); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2061 |
jccb (Assembler::notZero, DONE_LABEL); |
27608 | 2062 |
movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList))); |
2063 |
orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq))); |
|
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2064 |
jccb (Assembler::notZero, CheckSucc); |
27608 | 2065 |
movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2066 |
jmpb (DONE_LABEL); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2067 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2068 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2069 |
// The Following code fragment (EmitSync & 65536) improves the performance of |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2070 |
// contended applications and contended synchronization microbenchmarks. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2071 |
// Unfortunately the emission of the code - even though not executed - causes regressions |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2072 |
// in scimark and jetstream, evidently because of $ effects. Replacing the code |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2073 |
// with an equal number of never-executed NOPs results in the same regression. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2074 |
// We leave it off by default. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2075 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2076 |
if ((EmitSync & 65536) != 0) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2077 |
Label LSuccess, LGoSlowPath ; |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2078 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2079 |
bind (CheckSucc); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2080 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2081 |
// Optional pre-test ... it's safe to elide this |
30244 | 2082 |
cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), (int32_t)NULL_WORD); |
2083 |
jccb(Assembler::zero, LGoSlowPath); |
|
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2084 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2085 |
// We have a classic Dekker-style idiom: |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2086 |
// ST m->_owner = 0 ; MEMBAR; LD m->_succ |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2087 |
// There are a number of ways to implement the barrier: |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2088 |
// (1) lock:andl &m->_owner, 0 |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2089 |
// is fast, but mask doesn't currently support the "ANDL M,IMM32" form. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2090 |
// LOCK: ANDL [ebx+Offset(_Owner)-2], 0 |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2091 |
// Encodes as 81 31 OFF32 IMM32 or 83 63 OFF8 IMM8 |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2092 |
// (2) If supported, an explicit MFENCE is appealing. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2093 |
// In older IA32 processors MFENCE is slower than lock:add or xchg |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2094 |
// particularly if the write-buffer is full as might be the case if |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2095 |
// if stores closely precede the fence or fence-equivalent instruction. |
30244 | 2096 |
// See https://blogs.oracle.com/dave/entry/instruction_selection_for_volatile_fences |
2097 |
// as the situation has changed with Nehalem and Shanghai. |
|
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2098 |
// (3) In lieu of an explicit fence, use lock:addl to the top-of-stack |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2099 |
// The $lines underlying the top-of-stack should be in M-state. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2100 |
// The locked add instruction is serializing, of course. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2101 |
// (4) Use xchg, which is serializing |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2102 |
// mov boxReg, 0; xchgl boxReg, [tmpReg + Offset(_owner)-2] also works |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2103 |
// (5) ST m->_owner = 0 and then execute lock:orl &m->_succ, 0. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2104 |
// The integer condition codes will tell us if succ was 0. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2105 |
// Since _succ and _owner should reside in the same $line and |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2106 |
// we just stored into _owner, it's likely that the $line |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2107 |
// remains in M-state for the lock:orl. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2108 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2109 |
// We currently use (3), although it's likely that switching to (2) |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2110 |
// is correct for the future. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2111 |
|
27608 | 2112 |
movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2113 |
if (os::is_MP()) { |
30244 | 2114 |
lock(); addptr(Address(rsp, 0), 0); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2115 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2116 |
// Ratify _succ remains non-null |
27608 | 2117 |
cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), 0); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2118 |
jccb (Assembler::notZero, LSuccess); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2119 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2120 |
xorptr(boxReg, boxReg); // box is really EAX |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2121 |
if (os::is_MP()) { lock(); } |
27608 | 2122 |
cmpxchgptr(rsp, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); |
31782
b23b74f8ae8d
8130448: thread dump improvements, comment additions, new diagnostics inspired by 8077392
dcubed
parents:
31592
diff
changeset
|
2123 |
// There's no successor so we tried to regrab the lock with the |
b23b74f8ae8d
8130448: thread dump improvements, comment additions, new diagnostics inspired by 8077392
dcubed
parents:
31592
diff
changeset
|
2124 |
// placeholder value. If that didn't work, then another thread |
b23b74f8ae8d
8130448: thread dump improvements, comment additions, new diagnostics inspired by 8077392
dcubed
parents:
31592
diff
changeset
|
2125 |
// grabbed the lock so we're done (and exit was a success). |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2126 |
jccb (Assembler::notEqual, LSuccess); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2127 |
// Since we're low on registers we installed rsp as a placeholding in _owner. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2128 |
// Now install Self over rsp. This is safe as we're transitioning from |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2129 |
// non-null to non=null |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2130 |
get_thread (boxReg); |
27608 | 2131 |
movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), boxReg); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2132 |
// Intentional fall-through into LGoSlowPath ... |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2133 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2134 |
bind (LGoSlowPath); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2135 |
orptr(boxReg, 1); // set ICC.ZF=0 to indicate failure |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2136 |
jmpb (DONE_LABEL); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2137 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2138 |
bind (LSuccess); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2139 |
xorptr(boxReg, boxReg); // set ICC.ZF=1 to indicate success |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2140 |
jmpb (DONE_LABEL); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2141 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2142 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2143 |
bind (Stacked); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2144 |
// It's not inflated and it's not recursively stack-locked and it's not biased. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2145 |
// It must be stack-locked. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2146 |
// Try to reset the header to displaced header. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2147 |
// The "box" value on the stack is stable, so we can reload |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2148 |
// and be assured we observe the same value as above. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2149 |
movptr(tmpReg, Address(boxReg, 0)); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2150 |
if (os::is_MP()) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2151 |
lock(); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2152 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2153 |
cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2154 |
// Intention fall-thru into DONE_LABEL |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2155 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2156 |
// DONE_LABEL is a hot target - we'd really like to place it at the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2157 |
// start of cache line by padding with NOPs. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2158 |
// See the AMD and Intel software optimization manuals for the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2159 |
// most efficient "long" NOP encodings. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2160 |
// Unfortunately none of our alignment mechanisms suffice. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2161 |
if ((EmitSync & 65536) == 0) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2162 |
bind (CheckSucc); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2163 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2164 |
#else // _LP64 |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2165 |
// It's inflated |
30244 | 2166 |
if (EmitSync & 1024) { |
2167 |
// Emit code to check that _owner == Self |
|
2168 |
// We could fold the _owner test into subsequent code more efficiently |
|
2169 |
// than using a stand-alone check, but since _owner checking is off by |
|
2170 |
// default we don't bother. We also might consider predicating the |
|
2171 |
// _owner==Self check on Xcheck:jni or running on a debug build. |
|
2172 |
movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); |
|
2173 |
xorptr(boxReg, r15_thread); |
|
2174 |
} else { |
|
2175 |
xorptr(boxReg, boxReg); |
|
2176 |
} |
|
27608 | 2177 |
orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions))); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2178 |
jccb (Assembler::notZero, DONE_LABEL); |
27608 | 2179 |
movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq))); |
2180 |
orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList))); |
|
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2181 |
jccb (Assembler::notZero, CheckSucc); |
27608 | 2182 |
movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t)NULL_WORD); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2183 |
jmpb (DONE_LABEL); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2184 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2185 |
if ((EmitSync & 65536) == 0) { |
30244 | 2186 |
// Try to avoid passing control into the slow_path ... |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2187 |
Label LSuccess, LGoSlowPath ; |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2188 |
bind (CheckSucc); |
30244 | 2189 |
|
2190 |
// The following optional optimization can be elided if necessary |
|
2191 |
// Effectively: if (succ == null) goto SlowPath |
|
2192 |
// The code reduces the window for a race, however, |
|
2193 |
// and thus benefits performance. |
|
27608 | 2194 |
cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), (int32_t)NULL_WORD); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2195 |
jccb (Assembler::zero, LGoSlowPath); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2196 |
|
37251
9fc139ad74b5
8152358: code and comment cleanups found during the hunt for 8077392
dcubed
parents:
36561
diff
changeset
|
2197 |
xorptr(boxReg, boxReg); |
30244 | 2198 |
if ((EmitSync & 16) && os::is_MP()) { |
2199 |
xchgptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); |
|
2200 |
} else { |
|
2201 |
movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t)NULL_WORD); |
|
2202 |
if (os::is_MP()) { |
|
2203 |
// Memory barrier/fence |
|
2204 |
// Dekker pivot point -- fulcrum : ST Owner; MEMBAR; LD Succ |
|
2205 |
// Instead of MFENCE we use a dummy locked add of 0 to the top-of-stack. |
|
2206 |
// This is faster on Nehalem and AMD Shanghai/Barcelona. |
|
2207 |
// See https://blogs.oracle.com/dave/entry/instruction_selection_for_volatile_fences |
|
2208 |
// We might also restructure (ST Owner=0;barrier;LD _Succ) to |
|
2209 |
// (mov box,0; xchgq box, &m->Owner; LD _succ) . |
|
2210 |
lock(); addl(Address(rsp, 0), 0); |
|
2211 |
} |
|
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2212 |
} |
27608 | 2213 |
cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), (int32_t)NULL_WORD); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2214 |
jccb (Assembler::notZero, LSuccess); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2215 |
|
30244 | 2216 |
// Rare inopportune interleaving - race. |
2217 |
// The successor vanished in the small window above. |
|
2218 |
// The lock is contended -- (cxq|EntryList) != null -- and there's no apparent successor. |
|
2219 |
// We need to ensure progress and succession. |
|
2220 |
// Try to reacquire the lock. |
|
2221 |
// If that fails then the new owner is responsible for succession and this |
|
2222 |
// thread needs to take no further action and can exit via the fast path (success). |
|
2223 |
// If the re-acquire succeeds then pass control into the slow path. |
|
2224 |
// As implemented, this latter mode is horrible because we generated more |
|
2225 |
// coherence traffic on the lock *and* artifically extended the critical section |
|
2226 |
// length while by virtue of passing control into the slow path. |
|
2227 |
||
2228 |
// box is really RAX -- the following CMPXCHG depends on that binding |
|
2229 |
// cmpxchg R,[M] is equivalent to rax = CAS(M,rax,R) |
|
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2230 |
if (os::is_MP()) { lock(); } |
27608 | 2231 |
cmpxchgptr(r15_thread, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); |
31782
b23b74f8ae8d
8130448: thread dump improvements, comment additions, new diagnostics inspired by 8077392
dcubed
parents:
31592
diff
changeset
|
2232 |
// There's no successor so we tried to regrab the lock. |
b23b74f8ae8d
8130448: thread dump improvements, comment additions, new diagnostics inspired by 8077392
dcubed
parents:
31592
diff
changeset
|
2233 |
// If that didn't work, then another thread grabbed the |
b23b74f8ae8d
8130448: thread dump improvements, comment additions, new diagnostics inspired by 8077392
dcubed
parents:
31592
diff
changeset
|
2234 |
// lock so we're done (and exit was a success). |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2235 |
jccb (Assembler::notEqual, LSuccess); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2236 |
// Intentional fall-through into slow-path |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2237 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2238 |
bind (LGoSlowPath); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2239 |
orl (boxReg, 1); // set ICC.ZF=0 to indicate failure |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2240 |
jmpb (DONE_LABEL); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2241 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2242 |
bind (LSuccess); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2243 |
testl (boxReg, 0); // set ICC.ZF=1 to indicate success |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2244 |
jmpb (DONE_LABEL); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2245 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2246 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2247 |
bind (Stacked); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2248 |
movptr(tmpReg, Address (boxReg, 0)); // re-fetch |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2249 |
if (os::is_MP()) { lock(); } |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2250 |
cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2251 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2252 |
if (EmitSync & 65536) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2253 |
bind (CheckSucc); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2254 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2255 |
#endif |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2256 |
bind(DONE_LABEL); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2257 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2258 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2259 |
#endif // COMPILER2 |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2260 |
|
14626 | 2261 |
void MacroAssembler::c2bool(Register x) { |
2262 |
// implements x == 0 ? 0 : 1 |
|
2263 |
// note: must only look at least-significant byte of x |
|
2264 |
// since C-style booleans are stored in one byte |
|
2265 |
// only! (was bug) |
|
2266 |
andl(x, 0xFF); |
|
2267 |
setb(Assembler::notZero, x); |
|
2268 |
} |
|
2269 |
||
2270 |
// Wouldn't need if AddressLiteral version had new name |
|
2271 |
void MacroAssembler::call(Label& L, relocInfo::relocType rtype) { |
|
2272 |
Assembler::call(L, rtype); |
|
2273 |
} |
|
2274 |
||
2275 |
void MacroAssembler::call(Register entry) { |
|
2276 |
Assembler::call(entry); |
|
2277 |
} |
|
2278 |
||
2279 |
void MacroAssembler::call(AddressLiteral entry) { |
|
2280 |
if (reachable(entry)) { |
|
2281 |
Assembler::call_literal(entry.target(), entry.rspec()); |
|
2282 |
} else { |
|
2283 |
lea(rscratch1, entry); |
|
2284 |
Assembler::call(rscratch1); |
|
2285 |
} |
|
2286 |
} |
|
2287 |
||
35086
bbf32241d851
8072008: Emit direct call instead of linkTo* for recursive indy/MH.invoke* calls
vlivanov
parents:
34211
diff
changeset
|
2288 |
void MacroAssembler::ic_call(address entry, jint method_index) { |
bbf32241d851
8072008: Emit direct call instead of linkTo* for recursive indy/MH.invoke* calls
vlivanov
parents:
34211
diff
changeset
|
2289 |
RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index); |
14626 | 2290 |
movptr(rax, (intptr_t)Universe::non_oop_word()); |
2291 |
call(AddressLiteral(entry, rh)); |
|
2292 |
} |
|
2293 |
||
2294 |
// Implementation of call_VM versions |
|
2295 |
||
2296 |
void MacroAssembler::call_VM(Register oop_result, |
|
2297 |
address entry_point, |
|
2298 |
bool check_exceptions) { |
|
2299 |
Label C, E; |
|
2300 |
call(C, relocInfo::none); |
|
2301 |
jmp(E); |
|
2302 |
||
2303 |
bind(C); |
|
2304 |
call_VM_helper(oop_result, entry_point, 0, check_exceptions); |
|
2305 |
ret(0); |
|
2306 |
||
2307 |
bind(E); |
|
2308 |
} |
|
2309 |
||
2310 |
void MacroAssembler::call_VM(Register oop_result, |
|
2311 |
address entry_point, |
|
2312 |
Register arg_1, |
|
2313 |
bool check_exceptions) { |
|
2314 |
Label C, E; |
|
2315 |
call(C, relocInfo::none); |
|
2316 |
jmp(E); |
|
2317 |
||
2318 |
bind(C); |
|
2319 |
pass_arg1(this, arg_1); |
|
2320 |
call_VM_helper(oop_result, entry_point, 1, check_exceptions); |
|
2321 |
ret(0); |
|
2322 |
||
2323 |
bind(E); |
|
2324 |
} |
|
2325 |
||
2326 |
void MacroAssembler::call_VM(Register oop_result, |
|
2327 |
address entry_point, |
|
2328 |
Register arg_1, |
|
2329 |
Register arg_2, |
|
2330 |
bool check_exceptions) { |
|
2331 |
Label C, E; |
|
2332 |
call(C, relocInfo::none); |
|
2333 |
jmp(E); |
|
2334 |
||
2335 |
bind(C); |
|
2336 |
||
2337 |
LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); |
|
2338 |
||
2339 |
pass_arg2(this, arg_2); |
|
2340 |
pass_arg1(this, arg_1); |
|
2341 |
call_VM_helper(oop_result, entry_point, 2, check_exceptions); |
|
2342 |
ret(0); |
|
2343 |
||
2344 |
bind(E); |
|
2345 |
} |
|
2346 |
||
2347 |
void MacroAssembler::call_VM(Register oop_result, |
|
2348 |
address entry_point, |
|
2349 |
Register arg_1, |
|
2350 |
Register arg_2, |
|
2351 |
Register arg_3, |
|
2352 |
bool check_exceptions) { |
|
2353 |
Label C, E; |
|
2354 |
call(C, relocInfo::none); |
|
2355 |
jmp(E); |
|
2356 |
||
2357 |
bind(C); |
|
2358 |
||
2359 |
LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg")); |
|
2360 |
LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg")); |
|
2361 |
pass_arg3(this, arg_3); |
|
2362 |
||
2363 |
LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); |
|
2364 |
pass_arg2(this, arg_2); |
|
2365 |
||
2366 |
pass_arg1(this, arg_1); |
|
2367 |
call_VM_helper(oop_result, entry_point, 3, check_exceptions); |
|
2368 |
ret(0); |
|
2369 |
||
2370 |
bind(E); |
|
2371 |
} |
|
2372 |
||
2373 |
void MacroAssembler::call_VM(Register oop_result, |
|
2374 |
Register last_java_sp, |
|
2375 |
address entry_point, |
|
2376 |
int number_of_arguments, |
|
2377 |
bool check_exceptions) { |
|
2378 |
Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg); |
|
2379 |
call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions); |
|
2380 |
} |
|
2381 |
||
2382 |
void MacroAssembler::call_VM(Register oop_result, |
|
2383 |
Register last_java_sp, |
|
2384 |
address entry_point, |
|
2385 |
Register arg_1, |
|
2386 |
bool check_exceptions) { |
|
2387 |
pass_arg1(this, arg_1); |
|
2388 |
call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions); |
|
2389 |
} |
|
2390 |
||
2391 |
void MacroAssembler::call_VM(Register oop_result, |
|
2392 |
Register last_java_sp, |
|
2393 |
address entry_point, |
|
2394 |
Register arg_1, |
|
2395 |
Register arg_2, |
|
2396 |
bool check_exceptions) { |
|
2397 |
||
2398 |
LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); |
|
2399 |
pass_arg2(this, arg_2); |
|
2400 |
pass_arg1(this, arg_1); |
|
2401 |
call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions); |
|
2402 |
} |
|
2403 |
||
2404 |
void MacroAssembler::call_VM(Register oop_result, |
|
2405 |
Register last_java_sp, |
|
2406 |
address entry_point, |
|
2407 |
Register arg_1, |
|
2408 |
Register arg_2, |
|
2409 |
Register arg_3, |
|
2410 |
bool check_exceptions) { |
|
2411 |
LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg")); |
|
2412 |
LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg")); |
|
2413 |
pass_arg3(this, arg_3); |
|
2414 |
LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); |
|
2415 |
pass_arg2(this, arg_2); |
|
2416 |
pass_arg1(this, arg_1); |
|
2417 |
call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions); |
|
2418 |
} |
|
2419 |
||
2420 |
void MacroAssembler::super_call_VM(Register oop_result, |
|
2421 |
Register last_java_sp, |
|
2422 |
address entry_point, |
|
2423 |
int number_of_arguments, |
|
2424 |
bool check_exceptions) { |
|
2425 |
Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg); |
|
2426 |
MacroAssembler::call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions); |
|
2427 |
} |
|
2428 |
||
2429 |
void MacroAssembler::super_call_VM(Register oop_result, |
|
2430 |
Register last_java_sp, |
|
2431 |
address entry_point, |
|
2432 |
Register arg_1, |
|
2433 |
bool check_exceptions) { |
|
2434 |
pass_arg1(this, arg_1); |
|
2435 |
super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions); |
|
2436 |
} |
|
2437 |
||
2438 |
void MacroAssembler::super_call_VM(Register oop_result, |
|
2439 |
Register last_java_sp, |
|
2440 |
address entry_point, |
|
2441 |
Register arg_1, |
|
2442 |
Register arg_2, |
|
2443 |
bool check_exceptions) { |
|
2444 |
||
2445 |
LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); |
|
2446 |
pass_arg2(this, arg_2); |
|
2447 |
pass_arg1(this, arg_1); |
|
2448 |
super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions); |
|
2449 |
} |
|
2450 |
||
2451 |
void MacroAssembler::super_call_VM(Register oop_result, |
|
2452 |
Register last_java_sp, |
|
2453 |
address entry_point, |
|
2454 |
Register arg_1, |
|
2455 |
Register arg_2, |
|
2456 |
Register arg_3, |
|
2457 |
bool check_exceptions) { |
|
2458 |
LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg")); |
|
2459 |
LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg")); |
|
2460 |
pass_arg3(this, arg_3); |
|
2461 |
LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); |
|
2462 |
pass_arg2(this, arg_2); |
|
2463 |
pass_arg1(this, arg_1); |
|
2464 |
super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions); |
|
2465 |
} |
|
2466 |
||
2467 |
void MacroAssembler::call_VM_base(Register oop_result, |
|
2468 |
Register java_thread, |
|
2469 |
Register last_java_sp, |
|
2470 |
address entry_point, |
|
2471 |
int number_of_arguments, |
|
2472 |
bool check_exceptions) { |
|
2473 |
// determine java_thread register |
|
2474 |
if (!java_thread->is_valid()) { |
|
2475 |
#ifdef _LP64 |
|
2476 |
java_thread = r15_thread; |
|
2477 |
#else |
|
2478 |
java_thread = rdi; |
|
2479 |
get_thread(java_thread); |
|
2480 |
#endif // LP64 |
|
2481 |
} |
|
2482 |
// determine last_java_sp register |
|
2483 |
if (!last_java_sp->is_valid()) { |
|
2484 |
last_java_sp = rsp; |
|
2485 |
} |
|
2486 |
// debugging support |
|
2487 |
assert(number_of_arguments >= 0 , "cannot have negative number of arguments"); |
|
2488 |
LP64_ONLY(assert(java_thread == r15_thread, "unexpected register")); |
|
2489 |
#ifdef ASSERT |
|
2490 |
// TraceBytecodes does not use r12 but saves it over the call, so don't verify |
|
2491 |
// r12 is the heapbase. |
|
19979
ebe1dbb6e1aa
8015107: NPG: Use consistent naming for metaspace concepts
ehelin
parents:
19319
diff
changeset
|
2492 |
LP64_ONLY(if ((UseCompressedOops || UseCompressedClassPointers) && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");) |
14626 | 2493 |
#endif // ASSERT |
2494 |
||
2495 |
assert(java_thread != oop_result , "cannot use the same register for java_thread & oop_result"); |
|
2496 |
assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp"); |
|
2497 |
||
2498 |
// push java thread (becomes first argument of C function) |
|
2499 |
||
2500 |
NOT_LP64(push(java_thread); number_of_arguments++); |
|
2501 |
LP64_ONLY(mov(c_rarg0, r15_thread)); |
|
2502 |
||
2503 |
// set last Java frame before call |
|
2504 |
assert(last_java_sp != rbp, "can't use ebp/rbp"); |
|
2505 |
||
2506 |
// Only interpreter should have to set fp |
|
2507 |
set_last_Java_frame(java_thread, last_java_sp, rbp, NULL); |
|
2508 |
||
2509 |
// do the call, remove parameters |
|
2510 |
MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments); |
|
2511 |
||
2512 |
// restore the thread (cannot use the pushed argument since arguments |
|
2513 |
// may be overwritten by C code generated by an optimizing compiler); |
|
2514 |
// however can use the register value directly if it is callee saved. |
|
2515 |
if (LP64_ONLY(true ||) java_thread == rdi || java_thread == rsi) { |
|
2516 |
// rdi & rsi (also r15) are callee saved -> nothing to do |
|
2517 |
#ifdef ASSERT |
|
2518 |
guarantee(java_thread != rax, "change this code"); |
|
2519 |
push(rax); |
|
2520 |
{ Label L; |
|
2521 |
get_thread(rax); |
|
2522 |
cmpptr(java_thread, rax); |
|
2523 |
jcc(Assembler::equal, L); |
|
2524 |
STOP("MacroAssembler::call_VM_base: rdi not callee saved?"); |
|
2525 |
bind(L); |
|
2526 |
} |
|
2527 |
pop(rax); |
|
2528 |
#endif |
|
2529 |
} else { |
|
2530 |
get_thread(java_thread); |
|
2531 |
} |
|
2532 |
// reset last Java frame |
|
2533 |
// Only interpreter should have to clear fp |
|
2534 |
reset_last_Java_frame(java_thread, true, false); |
|
2535 |
||
2536 |
// C++ interp handles this in the interpreter |
|
2537 |
check_and_handle_popframe(java_thread); |
|
2538 |
check_and_handle_earlyret(java_thread); |
|
2539 |
||
2540 |
if (check_exceptions) { |
|
2541 |
// check for pending exceptions (java_thread is set upon return) |
|
2542 |
cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t) NULL_WORD); |
|
2543 |
#ifndef _LP64 |
|
2544 |
jump_cc(Assembler::notEqual, |
|
2545 |
RuntimeAddress(StubRoutines::forward_exception_entry())); |
|
2546 |
#else |
|
2547 |
// This used to conditionally jump to forward_exception however it is |
|
2548 |
// possible if we relocate that the branch will not reach. So we must jump |
|
2549 |
// around so we can always reach |
|
2550 |
||
2551 |
Label ok; |
|
2552 |
jcc(Assembler::equal, ok); |
|
2553 |
jump(RuntimeAddress(StubRoutines::forward_exception_entry())); |
|
2554 |
bind(ok); |
|
2555 |
#endif // LP64 |
|
2556 |
} |
|
2557 |
||
2558 |
// get oop result if there is one and reset the value in the thread |
|
2559 |
if (oop_result->is_valid()) { |
|
2560 |
get_vm_result(oop_result, java_thread); |
|
2561 |
} |
|
2562 |
} |
|
2563 |
||
2564 |
void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) { |
|
2565 |
||
2566 |
// Calculate the value for last_Java_sp |
|
2567 |
// somewhat subtle. call_VM does an intermediate call |
|
2568 |
// which places a return address on the stack just under the |
|
2569 |
// stack pointer as the user finsihed with it. This allows |
|
2570 |
// use to retrieve last_Java_pc from last_Java_sp[-1]. |
|
2571 |
// On 32bit we then have to push additional args on the stack to accomplish |
|
2572 |
// the actual requested call. On 64bit call_VM only can use register args |
|
2573 |
// so the only extra space is the return address that call_VM created. |
|
2574 |
// This hopefully explains the calculations here. |
|
2575 |
||
2576 |
#ifdef _LP64 |
|
2577 |
// We've pushed one address, correct last_Java_sp |
|
2578 |
lea(rax, Address(rsp, wordSize)); |
|
2579 |
#else |
|
2580 |
lea(rax, Address(rsp, (1 + number_of_arguments) * wordSize)); |
|
2581 |
#endif // LP64 |
|
2582 |
||
2583 |
call_VM_base(oop_result, noreg, rax, entry_point, number_of_arguments, check_exceptions); |
|
2584 |
||
2585 |
} |
|
2586 |
||
2587 |
void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) { |
|
2588 |
call_VM_leaf_base(entry_point, number_of_arguments); |
|
2589 |
} |
|
2590 |
||
2591 |
void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) { |
|
2592 |
pass_arg0(this, arg_0); |
|
2593 |
call_VM_leaf(entry_point, 1); |
|
2594 |
} |
|
2595 |
||
2596 |
void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) { |
|
2597 |
||
2598 |
LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg")); |
|
2599 |
pass_arg1(this, arg_1); |
|
2600 |
pass_arg0(this, arg_0); |
|
2601 |
call_VM_leaf(entry_point, 2); |
|
2602 |
} |
|
2603 |
||
2604 |
void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) { |
|
2605 |
LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg")); |
|
2606 |
LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); |
|
2607 |
pass_arg2(this, arg_2); |
|
2608 |
LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg")); |
|
2609 |
pass_arg1(this, arg_1); |
|
2610 |
pass_arg0(this, arg_0); |
|
2611 |
call_VM_leaf(entry_point, 3); |
|
2612 |
} |
|
2613 |
||
2614 |
void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) { |
|
2615 |
pass_arg0(this, arg_0); |
|
2616 |
MacroAssembler::call_VM_leaf_base(entry_point, 1); |
|
2617 |
} |
|
2618 |
||
2619 |
void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) { |
|
2620 |
||
2621 |
LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg")); |
|
2622 |
pass_arg1(this, arg_1); |
|
2623 |
pass_arg0(this, arg_0); |
|
2624 |
MacroAssembler::call_VM_leaf_base(entry_point, 2); |
|
2625 |
} |
|
2626 |
||
2627 |
void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) { |
|
2628 |
LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg")); |
|
2629 |
LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); |
|
2630 |
pass_arg2(this, arg_2); |
|
2631 |
LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg")); |
|
2632 |
pass_arg1(this, arg_1); |
|
2633 |
pass_arg0(this, arg_0); |
|
2634 |
MacroAssembler::call_VM_leaf_base(entry_point, 3); |
|
2635 |
} |
|
2636 |
||
2637 |
void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) { |
|
2638 |
LP64_ONLY(assert(arg_0 != c_rarg3, "smashed arg")); |
|
2639 |
LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg")); |
|
2640 |
LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg")); |
|
2641 |
pass_arg3(this, arg_3); |
|
2642 |
LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg")); |
|
2643 |
LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); |
|
2644 |
pass_arg2(this, arg_2); |
|
2645 |
LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg")); |
|
2646 |
pass_arg1(this, arg_1); |
|
2647 |
pass_arg0(this, arg_0); |
|
2648 |
MacroAssembler::call_VM_leaf_base(entry_point, 4); |
|
2649 |
} |
|
2650 |
||
2651 |
void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) { |
|
2652 |
movptr(oop_result, Address(java_thread, JavaThread::vm_result_offset())); |
|
2653 |
movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD); |
|
2654 |
verify_oop(oop_result, "broken oop in call_VM_base"); |
|
2655 |
} |
|
2656 |
||
2657 |
void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) { |
|
2658 |
movptr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset())); |
|
2659 |
movptr(Address(java_thread, JavaThread::vm_result_2_offset()), NULL_WORD); |
|
2660 |
} |
|
2661 |
||
2662 |
void MacroAssembler::check_and_handle_earlyret(Register java_thread) { |
|
2663 |
} |
|
2664 |
||
2665 |
void MacroAssembler::check_and_handle_popframe(Register java_thread) { |
|
2666 |
} |
|
2667 |
||
2668 |
void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm) { |
|
2669 |
if (reachable(src1)) { |
|
2670 |
cmpl(as_Address(src1), imm); |
|
2671 |
} else { |
|
2672 |
lea(rscratch1, src1); |
|
2673 |
cmpl(Address(rscratch1, 0), imm); |
|
2674 |
} |
|
2675 |
} |
|
2676 |
||
2677 |
void MacroAssembler::cmp32(Register src1, AddressLiteral src2) { |
|
2678 |
assert(!src2.is_lval(), "use cmpptr"); |
|
2679 |
if (reachable(src2)) { |
|
2680 |
cmpl(src1, as_Address(src2)); |
|
2681 |
} else { |
|
2682 |
lea(rscratch1, src2); |
|
2683 |
cmpl(src1, Address(rscratch1, 0)); |
|
2684 |
} |
|
2685 |
} |
|
2686 |
||
2687 |
void MacroAssembler::cmp32(Register src1, int32_t imm) { |
|
2688 |
Assembler::cmpl(src1, imm); |
|
2689 |
} |
|
2690 |
||
2691 |
void MacroAssembler::cmp32(Register src1, Address src2) { |
|
2692 |
Assembler::cmpl(src1, src2); |
|
2693 |
} |
|
2694 |
||
2695 |
void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) { |
|
2696 |
ucomisd(opr1, opr2); |
|
2697 |
||
2698 |
Label L; |
|
2699 |
if (unordered_is_less) { |
|
2700 |
movl(dst, -1); |
|
2701 |
jcc(Assembler::parity, L); |
|
2702 |
jcc(Assembler::below , L); |
|
2703 |
movl(dst, 0); |
|
2704 |
jcc(Assembler::equal , L); |
|
2705 |
increment(dst); |
|
2706 |
} else { // unordered is greater |
|
2707 |
movl(dst, 1); |
|
2708 |
jcc(Assembler::parity, L); |
|
2709 |
jcc(Assembler::above , L); |
|
2710 |
movl(dst, 0); |
|
2711 |
jcc(Assembler::equal , L); |
|
2712 |
decrementl(dst); |
|
2713 |
} |
|
2714 |
bind(L); |
|
2715 |
} |
|
2716 |
||
2717 |
void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) { |
|
2718 |
ucomiss(opr1, opr2); |
|
2719 |
||
2720 |
Label L; |
|
2721 |
if (unordered_is_less) { |
|
2722 |
movl(dst, -1); |
|
2723 |
jcc(Assembler::parity, L); |
|
2724 |
jcc(Assembler::below , L); |
|
2725 |
movl(dst, 0); |
|
2726 |
jcc(Assembler::equal , L); |
|
2727 |
increment(dst); |
|
2728 |
} else { // unordered is greater |
|
2729 |
movl(dst, 1); |
|
2730 |
jcc(Assembler::parity, L); |
|
2731 |
jcc(Assembler::above , L); |
|
2732 |
movl(dst, 0); |
|
2733 |
jcc(Assembler::equal , L); |
|
2734 |
decrementl(dst); |
|
2735 |
} |
|
2736 |
bind(L); |
|
2737 |
} |
|
2738 |
||
2739 |
||
2740 |
void MacroAssembler::cmp8(AddressLiteral src1, int imm) { |
|
2741 |
if (reachable(src1)) { |
|
2742 |
cmpb(as_Address(src1), imm); |
|
2743 |
} else { |
|
2744 |
lea(rscratch1, src1); |
|
2745 |
cmpb(Address(rscratch1, 0), imm); |
|
2746 |
} |
|
2747 |
} |
|
2748 |
||
2749 |
void MacroAssembler::cmpptr(Register src1, AddressLiteral src2) { |
|
2750 |
#ifdef _LP64 |
|
2751 |
if (src2.is_lval()) { |
|
2752 |
movptr(rscratch1, src2); |
|
2753 |
Assembler::cmpq(src1, rscratch1); |
|
2754 |
} else if (reachable(src2)) { |
|
2755 |
cmpq(src1, as_Address(src2)); |
|
2756 |
} else { |
|
2757 |
lea(rscratch1, src2); |
|
2758 |
Assembler::cmpq(src1, Address(rscratch1, 0)); |
|
2759 |
} |
|
2760 |
#else |
|
2761 |
if (src2.is_lval()) { |
|
2762 |
cmp_literal32(src1, (int32_t) src2.target(), src2.rspec()); |
|
2763 |
} else { |
|
2764 |
cmpl(src1, as_Address(src2)); |
|
2765 |
} |
|
2766 |
#endif // _LP64 |
|
2767 |
} |
|
2768 |
||
2769 |
void MacroAssembler::cmpptr(Address src1, AddressLiteral src2) { |
|
2770 |
assert(src2.is_lval(), "not a mem-mem compare"); |
|
2771 |
#ifdef _LP64 |
|
2772 |
// moves src2's literal address |
|
2773 |
movptr(rscratch1, src2); |
|
2774 |
Assembler::cmpq(src1, rscratch1); |
|
2775 |
#else |
|
2776 |
cmp_literal32(src1, (int32_t) src2.target(), src2.rspec()); |
|
2777 |
#endif // _LP64 |
|
2778 |
} |
|
2779 |
||
2780 |
void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr) { |
|
2781 |
if (reachable(adr)) { |
|
2782 |
if (os::is_MP()) |
|
2783 |
lock(); |
|
2784 |
cmpxchgptr(reg, as_Address(adr)); |
|
2785 |
} else { |
|
2786 |
lea(rscratch1, adr); |
|
2787 |
if (os::is_MP()) |
|
2788 |
lock(); |
|
2789 |
cmpxchgptr(reg, Address(rscratch1, 0)); |
|
2790 |
} |
|
2791 |
} |
|
2792 |
||
2793 |
void MacroAssembler::cmpxchgptr(Register reg, Address adr) { |
|
2794 |
LP64_ONLY(cmpxchgq(reg, adr)) NOT_LP64(cmpxchgl(reg, adr)); |
|
2795 |
} |
|
2796 |
||
2797 |
void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src) { |
|
2798 |
if (reachable(src)) { |
|
2799 |
Assembler::comisd(dst, as_Address(src)); |
|
2800 |
} else { |
|
2801 |
lea(rscratch1, src); |
|
2802 |
Assembler::comisd(dst, Address(rscratch1, 0)); |
|
2803 |
} |
|
2804 |
} |
|
2805 |
||
2806 |
void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src) { |
|
2807 |
if (reachable(src)) { |
|
2808 |
Assembler::comiss(dst, as_Address(src)); |
|
2809 |
} else { |
|
2810 |
lea(rscratch1, src); |
|
2811 |
Assembler::comiss(dst, Address(rscratch1, 0)); |
|
2812 |
} |
|
2813 |
} |
|
2814 |
||
2815 |
||
2816 |
void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr) { |
|
2817 |
Condition negated_cond = negate_condition(cond); |
|
2818 |
Label L; |
|
2819 |
jcc(negated_cond, L); |
|
23491 | 2820 |
pushf(); // Preserve flags |
14626 | 2821 |
atomic_incl(counter_addr); |
23491 | 2822 |
popf(); |
14626 | 2823 |
bind(L); |
2824 |
} |
|
2825 |
||
2826 |
int MacroAssembler::corrected_idivl(Register reg) { |
|
2827 |
// Full implementation of Java idiv and irem; checks for |
|
2828 |
// special case as described in JVM spec., p.243 & p.271. |
|
2829 |
// The function returns the (pc) offset of the idivl |
|
2830 |
// instruction - may be needed for implicit exceptions. |
|
2831 |
// |
|
2832 |
// normal case special case |
|
2833 |
// |
|
2834 |
// input : rax,: dividend min_int |
|
2835 |
// reg: divisor (may not be rax,/rdx) -1 |
|
2836 |
// |
|
2837 |
// output: rax,: quotient (= rax, idiv reg) min_int |
|
2838 |
// rdx: remainder (= rax, irem reg) 0 |
|
2839 |
assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register"); |
|
2840 |
const int min_int = 0x80000000; |
|
2841 |
Label normal_case, special_case; |
|
2842 |
||
2843 |
// check for special case |
|
2844 |
cmpl(rax, min_int); |
|
2845 |
jcc(Assembler::notEqual, normal_case); |
|
2846 |
xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0) |
|
2847 |
cmpl(reg, -1); |
|
2848 |
jcc(Assembler::equal, special_case); |
|
2849 |
||
2850 |
// handle normal case |
|
2851 |
bind(normal_case); |
|
2852 |
cdql(); |
|
2853 |
int idivl_offset = offset(); |
|
2854 |
idivl(reg); |
|
2855 |
||
2856 |
// normal and special case exit |
|
2857 |
bind(special_case); |
|
2858 |
||
2859 |
return idivl_offset; |
|
2860 |
} |
|
2861 |
||
2862 |
||
2863 |
||
2864 |
void MacroAssembler::decrementl(Register reg, int value) { |
|
2865 |
if (value == min_jint) {subl(reg, value) ; return; } |
|
2866 |
if (value < 0) { incrementl(reg, -value); return; } |
|
2867 |
if (value == 0) { ; return; } |
|
2868 |
if (value == 1 && UseIncDec) { decl(reg) ; return; } |
|
2869 |
/* else */ { subl(reg, value) ; return; } |
|
2870 |
} |
|
2871 |
||
2872 |
void MacroAssembler::decrementl(Address dst, int value) { |
|
2873 |
if (value == min_jint) {subl(dst, value) ; return; } |
|
2874 |
if (value < 0) { incrementl(dst, -value); return; } |
|
2875 |
if (value == 0) { ; return; } |
|
2876 |
if (value == 1 && UseIncDec) { decl(dst) ; return; } |
|
2877 |
/* else */ { subl(dst, value) ; return; } |
|
2878 |
} |
|
2879 |
||
2880 |
void MacroAssembler::division_with_shift (Register reg, int shift_value) { |
|
2881 |
assert (shift_value > 0, "illegal shift value"); |
|
2882 |
Label _is_positive; |
|
2883 |
testl (reg, reg); |
|
2884 |
jcc (Assembler::positive, _is_positive); |
|
2885 |
int offset = (1 << shift_value) - 1 ; |
|
2886 |
||
2887 |
if (offset == 1) { |
|
2888 |
incrementl(reg); |
|
2889 |
} else { |
|
2890 |
addl(reg, offset); |
|
2891 |
} |
|
2892 |
||
2893 |
bind (_is_positive); |
|
2894 |
sarl(reg, shift_value); |
|
2895 |
} |
|
2896 |
||
2897 |
void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src) { |
|
2898 |
if (reachable(src)) { |
|
2899 |
Assembler::divsd(dst, as_Address(src)); |
|
2900 |
} else { |
|
2901 |
lea(rscratch1, src); |
|
2902 |
Assembler::divsd(dst, Address(rscratch1, 0)); |
|
2903 |
} |
|
2904 |
} |
|
2905 |
||
2906 |
void MacroAssembler::divss(XMMRegister dst, AddressLiteral src) { |
|
2907 |
if (reachable(src)) { |
|
2908 |
Assembler::divss(dst, as_Address(src)); |
|
2909 |
} else { |
|
2910 |
lea(rscratch1, src); |
|
2911 |
Assembler::divss(dst, Address(rscratch1, 0)); |
|
2912 |
} |
|
2913 |
} |
|
2914 |
||
2915 |
// !defined(COMPILER2) is because of stupid core builds |
|
33160
c59f1676d27e
8136421: JEP 243: Java-Level JVM Compiler Interface
twisti
parents:
33089
diff
changeset
|
2916 |
#if !defined(_LP64) || defined(COMPILER1) || !defined(COMPILER2) || INCLUDE_JVMCI |
14626 | 2917 |
void MacroAssembler::empty_FPU_stack() { |
2918 |
if (VM_Version::supports_mmx()) { |
|
2919 |
emms(); |
|
2920 |
} else { |
|
2921 |
for (int i = 8; i-- > 0; ) ffree(i); |
|
2922 |
} |
|
2923 |
} |
|
33160
c59f1676d27e
8136421: JEP 243: Java-Level JVM Compiler Interface
twisti
parents:
33089
diff
changeset
|
2924 |
#endif // !LP64 || C1 || !C2 || INCLUDE_JVMCI |
14626 | 2925 |
|
2926 |
||
2927 |
// Defines obj, preserves var_size_in_bytes |
|
2928 |
void MacroAssembler::eden_allocate(Register obj, |
|
2929 |
Register var_size_in_bytes, |
|
2930 |
int con_size_in_bytes, |
|
2931 |
Register t1, |
|
2932 |
Label& slow_case) { |
|
2933 |
assert(obj == rax, "obj must be in rax, for cmpxchg"); |
|
2934 |
assert_different_registers(obj, var_size_in_bytes, t1); |
|
27625 | 2935 |
if (!Universe::heap()->supports_inline_contig_alloc()) { |
14626 | 2936 |
jmp(slow_case); |
2937 |
} else { |
|
2938 |
Register end = t1; |
|
2939 |
Label retry; |
|
2940 |
bind(retry); |
|
2941 |
ExternalAddress heap_top((address) Universe::heap()->top_addr()); |
|
2942 |
movptr(obj, heap_top); |
|
2943 |
if (var_size_in_bytes == noreg) { |
|
2944 |
lea(end, Address(obj, con_size_in_bytes)); |
|
2945 |
} else { |
|
2946 |
lea(end, Address(obj, var_size_in_bytes, Address::times_1)); |
|
2947 |
} |
|
2948 |
// if end < obj then we wrapped around => object too long => slow case |
|
2949 |
cmpptr(end, obj); |
|
2950 |
jcc(Assembler::below, slow_case); |
|
2951 |
cmpptr(end, ExternalAddress((address) Universe::heap()->end_addr())); |
|
2952 |
jcc(Assembler::above, slow_case); |
|
2953 |
// Compare obj with the top addr, and if still equal, store the new top addr in |
|
2954 |
// end at the address of the top addr pointer. Sets ZF if was equal, and clears |
|
2955 |
// it otherwise. Use lock prefix for atomicity on MPs. |
|
2956 |
locked_cmpxchgptr(end, heap_top); |
|
2957 |
jcc(Assembler::notEqual, retry); |
|
2958 |
} |
|
2959 |
} |
|
2960 |
||
2961 |
void MacroAssembler::enter() { |
|
2962 |
push(rbp); |
|
2963 |
mov(rbp, rsp); |
|
2964 |
} |
|
2965 |
||
2966 |
// A 5 byte nop that is safe for patching (see patch_verified_entry) |
|
2967 |
void MacroAssembler::fat_nop() { |
|
2968 |
if (UseAddressNop) { |
|
2969 |
addr_nop_5(); |
|
2970 |
} else { |
|
14837
a75c3082d106
8004250: replace AbstractAssembler a_byte/a_long with emit_int8/emit_int32
twisti
parents:
14834
diff
changeset
|
2971 |
emit_int8(0x26); // es: |
a75c3082d106
8004250: replace AbstractAssembler a_byte/a_long with emit_int8/emit_int32
twisti
parents:
14834
diff
changeset
|
2972 |
emit_int8(0x2e); // cs: |
a75c3082d106
8004250: replace AbstractAssembler a_byte/a_long with emit_int8/emit_int32
twisti
parents:
14834
diff
changeset
|
2973 |
emit_int8(0x64); // fs: |
a75c3082d106
8004250: replace AbstractAssembler a_byte/a_long with emit_int8/emit_int32
twisti
parents:
14834
diff
changeset
|
2974 |
emit_int8(0x65); // gs: |
a75c3082d106
8004250: replace AbstractAssembler a_byte/a_long with emit_int8/emit_int32
twisti
parents:
14834
diff
changeset
|
2975 |
emit_int8((unsigned char)0x90); |
14626 | 2976 |
} |
2977 |
} |
|
2978 |
||
2979 |
void MacroAssembler::fcmp(Register tmp) { |
|
2980 |
fcmp(tmp, 1, true, true); |
|
2981 |
} |
|
2982 |
||
2983 |
void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) { |
|
2984 |
assert(!pop_right || pop_left, "usage error"); |
|
2985 |
if (VM_Version::supports_cmov()) { |
|
2986 |
assert(tmp == noreg, "unneeded temp"); |
|
2987 |
if (pop_left) { |
|
2988 |
fucomip(index); |
|
2989 |
} else { |
|
2990 |
fucomi(index); |
|
2991 |
} |
|
2992 |
if (pop_right) { |
|
2993 |
fpop(); |
|
2994 |
} |
|
2995 |
} else { |
|
2996 |
assert(tmp != noreg, "need temp"); |
|
2997 |
if (pop_left) { |
|
2998 |
if (pop_right) { |
|
2999 |
fcompp(); |
|
3000 |
} else { |
|
3001 |
fcomp(index); |
|
3002 |
} |
|
3003 |
} else { |
|
3004 |
fcom(index); |
|
3005 |
} |
|
3006 |
// convert FPU condition into eflags condition via rax, |
|
3007 |
save_rax(tmp); |
|
3008 |
fwait(); fnstsw_ax(); |
|
3009 |
sahf(); |
|
3010 |
restore_rax(tmp); |
|
3011 |
} |
|
3012 |
// condition codes set as follows: |
|
3013 |
// |
|
3014 |
// CF (corresponds to C0) if x < y |
|
3015 |
// PF (corresponds to C2) if unordered |
|
3016 |
// ZF (corresponds to C3) if x = y |
|
3017 |
} |
|
3018 |
||
3019 |
void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less) { |
|
3020 |
fcmp2int(dst, unordered_is_less, 1, true, true); |
|
3021 |
} |
|
3022 |
||
3023 |
void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right) { |
|
3024 |
fcmp(VM_Version::supports_cmov() ? noreg : dst, index, pop_left, pop_right); |
|
3025 |
Label L; |
|
3026 |
if (unordered_is_less) { |
|
3027 |
movl(dst, -1); |
|
3028 |
jcc(Assembler::parity, L); |
|
3029 |
jcc(Assembler::below , L); |
|
3030 |
movl(dst, 0); |
|
3031 |
jcc(Assembler::equal , L); |
|
3032 |
increment(dst); |
|
3033 |
} else { // unordered is greater |
|
3034 |
movl(dst, 1); |
|
3035 |
jcc(Assembler::parity, L); |
|
3036 |
jcc(Assembler::above , L); |
|
3037 |
movl(dst, 0); |
|
3038 |
jcc(Assembler::equal , L); |
|
3039 |
decrementl(dst); |
|
3040 |
} |
|
3041 |
bind(L); |
|
3042 |
} |
|
3043 |
||
3044 |
void MacroAssembler::fld_d(AddressLiteral src) { |
|
3045 |
fld_d(as_Address(src)); |
|
3046 |
} |
|
3047 |
||
3048 |
void MacroAssembler::fld_s(AddressLiteral src) { |
|
3049 |
fld_s(as_Address(src)); |
|
3050 |
} |
|
3051 |
||
3052 |
void MacroAssembler::fld_x(AddressLiteral src) { |
|
3053 |
Assembler::fld_x(as_Address(src)); |
|
3054 |
} |
|
3055 |
||
3056 |
void MacroAssembler::fldcw(AddressLiteral src) { |
|
3057 |
Assembler::fldcw(as_Address(src)); |
|
3058 |
} |
|
3059 |
||
33089 | 3060 |
void MacroAssembler::mulpd(XMMRegister dst, AddressLiteral src) { |
3061 |
if (reachable(src)) { |
|
3062 |
Assembler::mulpd(dst, as_Address(src)); |
|
3063 |
} else { |
|
3064 |
lea(rscratch1, src); |
|
3065 |
Assembler::mulpd(dst, Address(rscratch1, 0)); |
|
3066 |
} |
|
3067 |
} |
|
3068 |
||
14626 | 3069 |
void MacroAssembler::increase_precision() { |
3070 |
subptr(rsp, BytesPerWord); |
|
3071 |
fnstcw(Address(rsp, 0)); |
|
3072 |
movl(rax, Address(rsp, 0)); |
|
3073 |
orl(rax, 0x300); |
|
3074 |
push(rax); |
|
3075 |
fldcw(Address(rsp, 0)); |
|
3076 |
pop(rax); |
|
3077 |
} |
|
3078 |
||
3079 |
void MacroAssembler::restore_precision() { |
|
3080 |
fldcw(Address(rsp, 0)); |
|
3081 |
addptr(rsp, BytesPerWord); |
|
3082 |
} |
|
3083 |
||
3084 |
void MacroAssembler::fpop() { |
|
3085 |
ffree(); |
|
3086 |
fincstp(); |
|
3087 |
} |
|
3088 |
||
32391
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3089 |
void MacroAssembler::load_float(Address src) { |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3090 |
if (UseSSE >= 1) { |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3091 |
movflt(xmm0, src); |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3092 |
} else { |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3093 |
LP64_ONLY(ShouldNotReachHere()); |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3094 |
NOT_LP64(fld_s(src)); |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3095 |
} |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3096 |
} |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3097 |
|
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3098 |
void MacroAssembler::store_float(Address dst) { |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3099 |
if (UseSSE >= 1) { |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3100 |
movflt(dst, xmm0); |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3101 |
} else { |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3102 |
LP64_ONLY(ShouldNotReachHere()); |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3103 |
NOT_LP64(fstp_s(dst)); |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3104 |
} |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3105 |
} |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3106 |
|
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3107 |
void MacroAssembler::load_double(Address src) { |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3108 |
if (UseSSE >= 2) { |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3109 |
movdbl(xmm0, src); |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3110 |
} else { |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3111 |
LP64_ONLY(ShouldNotReachHere()); |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3112 |
NOT_LP64(fld_d(src)); |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3113 |
} |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3114 |
} |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3115 |
|
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3116 |
void MacroAssembler::store_double(Address dst) { |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3117 |
if (UseSSE >= 2) { |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3118 |
movdbl(dst, xmm0); |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3119 |
} else { |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3120 |
LP64_ONLY(ShouldNotReachHere()); |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3121 |
NOT_LP64(fstp_d(dst)); |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3122 |
} |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3123 |
} |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
3124 |
|
14626 | 3125 |
void MacroAssembler::fremr(Register tmp) { |
3126 |
save_rax(tmp); |
|
3127 |
{ Label L; |
|
3128 |
bind(L); |
|
3129 |
fprem(); |
|
3130 |
fwait(); fnstsw_ax(); |
|
3131 |
#ifdef _LP64 |
|
3132 |
testl(rax, 0x400); |
|
3133 |
jcc(Assembler::notEqual, L); |
|
3134 |
#else |
|
3135 |
sahf(); |
|
3136 |
jcc(Assembler::parity, L); |
|
3137 |
#endif // _LP64 |
|
3138 |
} |
|
3139 |
restore_rax(tmp); |
|
3140 |
// Result is in ST0. |
|
3141 |
// Note: fxch & fpop to get rid of ST1 |
|
3142 |
// (otherwise FPU stack could overflow eventually) |
|
3143 |
fxch(1); |
|
3144 |
fpop(); |
|
3145 |
} |
|
3146 |
||
3147 |
||
3148 |
void MacroAssembler::incrementl(AddressLiteral dst) { |
|
3149 |
if (reachable(dst)) { |
|
3150 |
incrementl(as_Address(dst)); |
|
3151 |
} else { |
|
3152 |
lea(rscratch1, dst); |
|
3153 |
incrementl(Address(rscratch1, 0)); |
|
3154 |
} |
|
3155 |
} |
|
3156 |
||
3157 |
void MacroAssembler::incrementl(ArrayAddress dst) { |
|
3158 |
incrementl(as_Address(dst)); |
|
3159 |
} |
|
3160 |
||
3161 |
void MacroAssembler::incrementl(Register reg, int value) { |
|
3162 |
if (value == min_jint) {addl(reg, value) ; return; } |
|
3163 |
if (value < 0) { decrementl(reg, -value); return; } |
|
3164 |
if (value == 0) { ; return; } |
|
3165 |
if (value == 1 && UseIncDec) { incl(reg) ; return; } |
|
3166 |
/* else */ { addl(reg, value) ; return; } |
|
3167 |
} |
|
3168 |
||
3169 |
void MacroAssembler::incrementl(Address dst, int value) { |
|
3170 |
if (value == min_jint) {addl(dst, value) ; return; } |
|
3171 |
if (value < 0) { decrementl(dst, -value); return; } |
|
3172 |
if (value == 0) { ; return; } |
|
3173 |
if (value == 1 && UseIncDec) { incl(dst) ; return; } |
|
3174 |
/* else */ { addl(dst, value) ; return; } |
|
3175 |
} |
|
3176 |
||
3177 |
void MacroAssembler::jump(AddressLiteral dst) { |
|
3178 |
if (reachable(dst)) { |
|
3179 |
jmp_literal(dst.target(), dst.rspec()); |
|
3180 |
} else { |
|
3181 |
lea(rscratch1, dst); |
|
3182 |
jmp(rscratch1); |
|
3183 |
} |
|
3184 |
} |
|
3185 |
||
3186 |
void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst) { |
|
3187 |
if (reachable(dst)) { |
|
3188 |
InstructionMark im(this); |
|
3189 |
relocate(dst.reloc()); |
|
3190 |
const int short_size = 2; |
|
3191 |
const int long_size = 6; |
|
3192 |
int offs = (intptr_t)dst.target() - ((intptr_t)pc()); |
|
3193 |
if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) { |
|
3194 |
// 0111 tttn #8-bit disp |
|
14837
a75c3082d106
8004250: replace AbstractAssembler a_byte/a_long with emit_int8/emit_int32
twisti
parents:
14834
diff
changeset
|
3195 |
emit_int8(0x70 | cc); |
a75c3082d106
8004250: replace AbstractAssembler a_byte/a_long with emit_int8/emit_int32
twisti
parents:
14834
diff
changeset
|
3196 |
emit_int8((offs - short_size) & 0xFF); |
14626 | 3197 |
} else { |
3198 |
// 0000 1111 1000 tttn #32-bit disp |
|
14837
a75c3082d106
8004250: replace AbstractAssembler a_byte/a_long with emit_int8/emit_int32
twisti
parents:
14834
diff
changeset
|
3199 |
emit_int8(0x0F); |
a75c3082d106
8004250: replace AbstractAssembler a_byte/a_long with emit_int8/emit_int32
twisti
parents:
14834
diff
changeset
|
3200 |
emit_int8((unsigned char)(0x80 | cc)); |
15116
af423dcb739c
8004537: replace AbstractAssembler emit_long with emit_int32
twisti
parents:
15115
diff
changeset
|
3201 |
emit_int32(offs - long_size); |
14626 | 3202 |
} |
3203 |
} else { |
|
3204 |
#ifdef ASSERT |
|
3205 |
warning("reversing conditional branch"); |
|
3206 |
#endif /* ASSERT */ |
|
3207 |
Label skip; |
|
3208 |
jccb(reverse[cc], skip); |
|
3209 |
lea(rscratch1, dst); |
|
3210 |
Assembler::jmp(rscratch1); |
|
3211 |
bind(skip); |
|
3212 |
} |
|
3213 |
} |
|
3214 |
||
3215 |
void MacroAssembler::ldmxcsr(AddressLiteral src) { |
|
3216 |
if (reachable(src)) { |
|
3217 |
Assembler::ldmxcsr(as_Address(src)); |
|
3218 |
} else { |
|
3219 |
lea(rscratch1, src); |
|
3220 |
Assembler::ldmxcsr(Address(rscratch1, 0)); |
|
3221 |
} |
|
3222 |
} |
|
3223 |
||
3224 |
int MacroAssembler::load_signed_byte(Register dst, Address src) { |
|
3225 |
int off; |
|
3226 |
if (LP64_ONLY(true ||) VM_Version::is_P6()) { |
|
3227 |
off = offset(); |
|
3228 |
movsbl(dst, src); // movsxb |
|
3229 |
} else { |
|
3230 |
off = load_unsigned_byte(dst, src); |
|
3231 |
shll(dst, 24); |
|
3232 |
sarl(dst, 24); |
|
3233 |
} |
|
3234 |
return off; |
|
3235 |
} |
|
3236 |
||
3237 |
// Note: load_signed_short used to be called load_signed_word. |
|
3238 |
// Although the 'w' in x86 opcodes refers to the term "word" in the assembler |
|
3239 |
// manual, which means 16 bits, that usage is found nowhere in HotSpot code. |
|
3240 |
// The term "word" in HotSpot means a 32- or 64-bit machine word. |
|
3241 |
int MacroAssembler::load_signed_short(Register dst, Address src) { |
|
3242 |
int off; |
|
3243 |
if (LP64_ONLY(true ||) VM_Version::is_P6()) { |
|
3244 |
// This is dubious to me since it seems safe to do a signed 16 => 64 bit |
|
3245 |
// version but this is what 64bit has always done. This seems to imply |
|
3246 |
// that users are only using 32bits worth. |
|
3247 |
off = offset(); |
|
3248 |
movswl(dst, src); // movsxw |
|
3249 |
} else { |
|
3250 |
off = load_unsigned_short(dst, src); |
|
3251 |
shll(dst, 16); |
|
3252 |
sarl(dst, 16); |
|
3253 |
} |
|
3254 |
return off; |
|
3255 |
} |
|
3256 |
||
3257 |
int MacroAssembler::load_unsigned_byte(Register dst, Address src) { |
|
3258 |
// According to Intel Doc. AP-526, "Zero-Extension of Short", p.16, |
|
3259 |
// and "3.9 Partial Register Penalties", p. 22). |
|
3260 |
int off; |
|
3261 |
if (LP64_ONLY(true || ) VM_Version::is_P6() || src.uses(dst)) { |
|
3262 |
off = offset(); |
|
3263 |
movzbl(dst, src); // movzxb |
|
3264 |
} else { |
|
3265 |
xorl(dst, dst); |
|
3266 |
off = offset(); |
|
3267 |
movb(dst, src); |
|
3268 |
} |
|
3269 |
return off; |
|
3270 |
} |
|
3271 |
||
3272 |
// Note: load_unsigned_short used to be called load_unsigned_word. |
|
3273 |
int MacroAssembler::load_unsigned_short(Register dst, Address src) { |
|
3274 |
// According to Intel Doc. AP-526, "Zero-Extension of Short", p.16, |
|
3275 |
// and "3.9 Partial Register Penalties", p. 22). |
|
3276 |
int off; |
|
3277 |
if (LP64_ONLY(true ||) VM_Version::is_P6() || src.uses(dst)) { |
|
3278 |
off = offset(); |
|
3279 |
movzwl(dst, src); // movzxw |
|
3280 |
} else { |
|
3281 |
xorl(dst, dst); |
|
3282 |
off = offset(); |
|
3283 |
movw(dst, src); |
|
3284 |
} |
|
3285 |
return off; |
|
3286 |
} |
|
3287 |
||
3288 |
void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) { |
|
3289 |
switch (size_in_bytes) { |
|
3290 |
#ifndef _LP64 |
|
3291 |
case 8: |
|
3292 |
assert(dst2 != noreg, "second dest register required"); |
|
3293 |
movl(dst, src); |
|
3294 |
movl(dst2, src.plus_disp(BytesPerInt)); |
|
3295 |
break; |
|
3296 |
#else |
|
3297 |
case 8: movq(dst, src); break; |
|
3298 |
#endif |
|
3299 |
case 4: movl(dst, src); break; |
|
3300 |
case 2: is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break; |
|
3301 |
case 1: is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break; |
|
3302 |
default: ShouldNotReachHere(); |
|
3303 |
} |
|
3304 |
} |
|
3305 |
||
3306 |
void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) { |
|
3307 |
switch (size_in_bytes) { |
|
3308 |
#ifndef _LP64 |
|
3309 |
case 8: |
|
3310 |
assert(src2 != noreg, "second source register required"); |
|
3311 |
movl(dst, src); |
|
3312 |
movl(dst.plus_disp(BytesPerInt), src2); |
|
3313 |
break; |
|
3314 |
#else |
|
3315 |
case 8: movq(dst, src); break; |
|
3316 |
#endif |
|
3317 |
case 4: movl(dst, src); break; |
|
3318 |
case 2: movw(dst, src); break; |
|
3319 |
case 1: movb(dst, src); break; |
|
3320 |
default: ShouldNotReachHere(); |
|
3321 |
} |
|
3322 |
} |
|
3323 |
||
3324 |
void MacroAssembler::mov32(AddressLiteral dst, Register src) { |
|
3325 |
if (reachable(dst)) { |
|
3326 |
movl(as_Address(dst), src); |
|
3327 |
} else { |
|
3328 |
lea(rscratch1, dst); |
|
3329 |
movl(Address(rscratch1, 0), src); |
|
3330 |
} |
|
3331 |
} |
|
3332 |
||
3333 |
void MacroAssembler::mov32(Register dst, AddressLiteral src) { |
|
3334 |
if (reachable(src)) { |
|
3335 |
movl(dst, as_Address(src)); |
|
3336 |
} else { |
|
3337 |
lea(rscratch1, src); |
|
3338 |
movl(dst, Address(rscratch1, 0)); |
|
3339 |
} |
|
3340 |
} |
|
3341 |
||
3342 |
// C++ bool manipulation |
|
3343 |
||
3344 |
void MacroAssembler::movbool(Register dst, Address src) { |
|
3345 |
if(sizeof(bool) == 1) |
|
3346 |
movb(dst, src); |
|
3347 |
else if(sizeof(bool) == 2) |
|
3348 |
movw(dst, src); |
|
3349 |
else if(sizeof(bool) == 4) |
|
3350 |
movl(dst, src); |
|
3351 |
else |
|
3352 |
// unsupported |
|
3353 |
ShouldNotReachHere(); |
|
3354 |
} |
|
3355 |
||
3356 |
void MacroAssembler::movbool(Address dst, bool boolconst) { |
|
3357 |
if(sizeof(bool) == 1) |
|
3358 |
movb(dst, (int) boolconst); |
|
3359 |
else if(sizeof(bool) == 2) |
|
3360 |
movw(dst, (int) boolconst); |
|
3361 |
else if(sizeof(bool) == 4) |
|
3362 |
movl(dst, (int) boolconst); |
|
3363 |
else |
|
3364 |
// unsupported |
|
3365 |
ShouldNotReachHere(); |
|
3366 |
} |
|
3367 |
||
3368 |
void MacroAssembler::movbool(Address dst, Register src) { |
|
3369 |
if(sizeof(bool) == 1) |
|
3370 |
movb(dst, src); |
|
3371 |
else if(sizeof(bool) == 2) |
|
3372 |
movw(dst, src); |
|
3373 |
else if(sizeof(bool) == 4) |
|
3374 |
movl(dst, src); |
|
3375 |
else |
|
3376 |
// unsupported |
|
3377 |
ShouldNotReachHere(); |
|
3378 |
} |
|
3379 |
||
3380 |
void MacroAssembler::movbyte(ArrayAddress dst, int src) { |
|
3381 |
movb(as_Address(dst), src); |
|
3382 |
} |
|
3383 |
||
3384 |
void MacroAssembler::movdl(XMMRegister dst, AddressLiteral src) { |
|
3385 |
if (reachable(src)) { |
|
3386 |
movdl(dst, as_Address(src)); |
|
3387 |
} else { |
|
3388 |
lea(rscratch1, src); |
|
3389 |
movdl(dst, Address(rscratch1, 0)); |
|
3390 |
} |
|
3391 |
} |
|
3392 |
||
3393 |
void MacroAssembler::movq(XMMRegister dst, AddressLiteral src) { |
|
3394 |
if (reachable(src)) { |
|
3395 |
movq(dst, as_Address(src)); |
|
3396 |
} else { |
|
3397 |
lea(rscratch1, src); |
|
3398 |
movq(dst, Address(rscratch1, 0)); |
|
3399 |
} |
|
3400 |
} |
|
3401 |
||
38049 | 3402 |
void MacroAssembler::setvectmask(Register dst, Register src) { |
3403 |
Assembler::movl(dst, 1); |
|
3404 |
Assembler::shlxl(dst, dst, src); |
|
3405 |
Assembler::decl(dst); |
|
3406 |
Assembler::kmovdl(k1, dst); |
|
3407 |
Assembler::movl(dst, src); |
|
3408 |
} |
|
3409 |
||
3410 |
void MacroAssembler::restorevectmask() { |
|
3411 |
Assembler::knotwl(k1, k0); |
|
3412 |
} |
|
3413 |
||
14626 | 3414 |
void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src) { |
3415 |
if (reachable(src)) { |
|
3416 |
if (UseXmmLoadAndClearUpper) { |
|
3417 |
movsd (dst, as_Address(src)); |
|
3418 |
} else { |
|
3419 |
movlpd(dst, as_Address(src)); |
|
3420 |
} |
|
3421 |
} else { |
|
3422 |
lea(rscratch1, src); |
|
3423 |
if (UseXmmLoadAndClearUpper) { |
|
3424 |
movsd (dst, Address(rscratch1, 0)); |
|
3425 |
} else { |
|
3426 |
movlpd(dst, Address(rscratch1, 0)); |
|
3427 |
} |
|
3428 |
} |
|
3429 |
} |
|
3430 |
||
3431 |
void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src) { |
|
3432 |
if (reachable(src)) { |
|
3433 |
movss(dst, as_Address(src)); |
|
3434 |
} else { |
|
3435 |
lea(rscratch1, src); |
|
3436 |
movss(dst, Address(rscratch1, 0)); |
|
3437 |
} |
|
3438 |
} |
|
3439 |
||
3440 |
void MacroAssembler::movptr(Register dst, Register src) { |
|
3441 |
LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src)); |
|
3442 |
} |
|
3443 |
||
3444 |
void MacroAssembler::movptr(Register dst, Address src) { |
|
3445 |
LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src)); |
|
3446 |
} |
|
3447 |
||
3448 |
// src should NEVER be a real pointer. Use AddressLiteral for true pointers |
|
3449 |
void MacroAssembler::movptr(Register dst, intptr_t src) { |
|
3450 |
LP64_ONLY(mov64(dst, src)) NOT_LP64(movl(dst, src)); |
|
3451 |
} |
|
3452 |
||
3453 |
void MacroAssembler::movptr(Address dst, Register src) { |
|
3454 |
LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src)); |
|
3455 |
} |
|
3456 |
||
34162 | 3457 |
void MacroAssembler::movdqu(Address dst, XMMRegister src) { |
3458 |
if (UseAVX > 2 && !VM_Version::supports_avx512vl() && (src->encoding() > 15)) { |
|
36561
b18243f4d955
8151002: Make Assembler methods vextract and vinsert match actual instructions
mikael
parents:
36554
diff
changeset
|
3459 |
Assembler::vextractf32x4(dst, src, 0); |
34162 | 3460 |
} else { |
3461 |
Assembler::movdqu(dst, src); |
|
3462 |
} |
|
3463 |
} |
|
3464 |
||
3465 |
void MacroAssembler::movdqu(XMMRegister dst, Address src) { |
|
3466 |
if (UseAVX > 2 && !VM_Version::supports_avx512vl() && (dst->encoding() > 15)) { |
|
36561
b18243f4d955
8151002: Make Assembler methods vextract and vinsert match actual instructions
mikael
parents:
36554
diff
changeset
|
3467 |
Assembler::vinsertf32x4(dst, dst, src, 0); |
34162 | 3468 |
} else { |
3469 |
Assembler::movdqu(dst, src); |
|
3470 |
} |
|
3471 |
} |
|
3472 |
||
3473 |
void MacroAssembler::movdqu(XMMRegister dst, XMMRegister src) { |
|
3474 |
if (UseAVX > 2 && !VM_Version::supports_avx512vl()) { |
|
3475 |
Assembler::evmovdqul(dst, src, Assembler::AVX_512bit); |
|
3476 |
} else { |
|
3477 |
Assembler::movdqu(dst, src); |
|
3478 |
} |
|
3479 |
} |
|
3480 |
||
14626 | 3481 |
void MacroAssembler::movdqu(XMMRegister dst, AddressLiteral src) { |
3482 |
if (reachable(src)) { |
|
34162 | 3483 |
movdqu(dst, as_Address(src)); |
14626 | 3484 |
} else { |
3485 |
lea(rscratch1, src); |
|
34162 | 3486 |
movdqu(dst, Address(rscratch1, 0)); |
3487 |
} |
|
3488 |
} |
|
3489 |
||
3490 |
void MacroAssembler::vmovdqu(Address dst, XMMRegister src) { |
|
3491 |
if (UseAVX > 2 && !VM_Version::supports_avx512vl() && (src->encoding() > 15)) { |
|
36561
b18243f4d955
8151002: Make Assembler methods vextract and vinsert match actual instructions
mikael
parents:
36554
diff
changeset
|
3492 |
vextractf64x4_low(dst, src); |
34162 | 3493 |
} else { |
3494 |
Assembler::vmovdqu(dst, src); |
|
3495 |
} |
|
3496 |
} |
|
3497 |
||
3498 |
void MacroAssembler::vmovdqu(XMMRegister dst, Address src) { |
|
3499 |
if (UseAVX > 2 && !VM_Version::supports_avx512vl() && (dst->encoding() > 15)) { |
|
36561
b18243f4d955
8151002: Make Assembler methods vextract and vinsert match actual instructions
mikael
parents:
36554
diff
changeset
|
3500 |
vinsertf64x4_low(dst, src); |
34162 | 3501 |
} else { |
3502 |
Assembler::vmovdqu(dst, src); |
|
3503 |
} |
|
3504 |
} |
|
3505 |
||
3506 |
void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) { |
|
3507 |
if (UseAVX > 2 && !VM_Version::supports_avx512vl()) { |
|
3508 |
Assembler::evmovdqul(dst, src, Assembler::AVX_512bit); |
|
3509 |
} |
|
3510 |
else { |
|
3511 |
Assembler::vmovdqu(dst, src); |
|
3512 |
} |
|
3513 |
} |
|
3514 |
||
3515 |
void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src) { |
|
3516 |
if (reachable(src)) { |
|
3517 |
vmovdqu(dst, as_Address(src)); |
|
3518 |
} |
|
3519 |
else { |
|
3520 |
lea(rscratch1, src); |
|
3521 |
vmovdqu(dst, Address(rscratch1, 0)); |
|
14626 | 3522 |
} |
3523 |
} |
|
3524 |
||
18507
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
3525 |
void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src) { |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
3526 |
if (reachable(src)) { |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
3527 |
Assembler::movdqa(dst, as_Address(src)); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
3528 |
} else { |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
3529 |
lea(rscratch1, src); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
3530 |
Assembler::movdqa(dst, Address(rscratch1, 0)); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
3531 |
} |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
3532 |
} |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
3533 |
|
14626 | 3534 |
void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src) { |
3535 |
if (reachable(src)) { |
|
3536 |
Assembler::movsd(dst, as_Address(src)); |
|
3537 |
} else { |
|
3538 |
lea(rscratch1, src); |
|
3539 |
Assembler::movsd(dst, Address(rscratch1, 0)); |
|
3540 |
} |
|
3541 |
} |
|
3542 |
||
3543 |
void MacroAssembler::movss(XMMRegister dst, AddressLiteral src) { |
|
3544 |
if (reachable(src)) { |
|
3545 |
Assembler::movss(dst, as_Address(src)); |
|
3546 |
} else { |
|
3547 |
lea(rscratch1, src); |
|
3548 |
Assembler::movss(dst, Address(rscratch1, 0)); |
|
3549 |
} |
|
3550 |
} |
|
3551 |
||
3552 |
void MacroAssembler::mulsd(XMMRegister dst, AddressLiteral src) { |
|
3553 |
if (reachable(src)) { |
|
3554 |
Assembler::mulsd(dst, as_Address(src)); |
|
3555 |
} else { |
|
3556 |
lea(rscratch1, src); |
|
3557 |
Assembler::mulsd(dst, Address(rscratch1, 0)); |
|
3558 |
} |
|
3559 |
} |
|
3560 |
||
3561 |
void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src) { |
|
3562 |
if (reachable(src)) { |
|
3563 |
Assembler::mulss(dst, as_Address(src)); |
|
3564 |
} else { |
|
3565 |
lea(rscratch1, src); |
|
3566 |
Assembler::mulss(dst, Address(rscratch1, 0)); |
|
3567 |
} |
|
3568 |
} |
|
3569 |
||
3570 |
void MacroAssembler::null_check(Register reg, int offset) { |
|
3571 |
if (needs_explicit_null_check(offset)) { |
|
3572 |
// provoke OS NULL exception if reg = NULL by |
|
3573 |
// accessing M[reg] w/o changing any (non-CC) registers |
|
3574 |
// NOTE: cmpl is plenty here to provoke a segv |
|
3575 |
cmpptr(rax, Address(reg, 0)); |
|
3576 |
// Note: should probably use testl(rax, Address(reg, 0)); |
|
3577 |
// may be shorter code (however, this version of |
|
3578 |
// testl needs to be implemented first) |
|
3579 |
} else { |
|
3580 |
// nothing to do, (later) access of M[reg + offset] |
|
3581 |
// will provoke OS NULL exception if reg = NULL |
|
3582 |
} |
|
3583 |
} |
|
3584 |
||
3585 |
void MacroAssembler::os_breakpoint() { |
|
3586 |
// instead of directly emitting a breakpoint, call os:breakpoint for better debugability |
|
3587 |
// (e.g., MSVC can't call ps() otherwise) |
|
3588 |
call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint))); |
|
3589 |
} |
|
3590 |
||
34162 | 3591 |
#ifdef _LP64 |
3592 |
#define XSTATE_BV 0x200 |
|
3593 |
#endif |
|
3594 |
||
14626 | 3595 |
void MacroAssembler::pop_CPU_state() { |
3596 |
pop_FPU_state(); |
|
3597 |
pop_IU_state(); |
|
3598 |
} |
|
3599 |
||
3600 |
void MacroAssembler::pop_FPU_state() { |
|
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
3601 |
#ifndef _LP64 |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
3602 |
frstor(Address(rsp, 0)); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
3603 |
#else |
34162 | 3604 |
fxrstor(Address(rsp, 0)); |
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
3605 |
#endif |
14626 | 3606 |
addptr(rsp, FPUStateSizeInWords * wordSize); |
3607 |
} |
|
3608 |
||
3609 |
void MacroAssembler::pop_IU_state() { |
|
3610 |
popa(); |
|
3611 |
LP64_ONLY(addq(rsp, 8)); |
|
3612 |
popf(); |
|
3613 |
} |
|
3614 |
||
3615 |
// Save Integer and Float state |
|
3616 |
// Warning: Stack must be 16 byte aligned (64bit) |
|
3617 |
void MacroAssembler::push_CPU_state() { |
|
3618 |
push_IU_state(); |
|
3619 |
push_FPU_state(); |
|
3620 |
} |
|
3621 |
||
3622 |
void MacroAssembler::push_FPU_state() { |
|
3623 |
subptr(rsp, FPUStateSizeInWords * wordSize); |
|
3624 |
#ifndef _LP64 |
|
3625 |
fnsave(Address(rsp, 0)); |
|
3626 |
fwait(); |
|
3627 |
#else |
|
34162 | 3628 |
fxsave(Address(rsp, 0)); |
14626 | 3629 |
#endif // LP64 |
3630 |
} |
|
3631 |
||
3632 |
void MacroAssembler::push_IU_state() { |
|
3633 |
// Push flags first because pusha kills them |
|
3634 |
pushf(); |
|
3635 |
// Make sure rsp stays 16-byte aligned |
|
3636 |
LP64_ONLY(subq(rsp, 8)); |
|
3637 |
pusha(); |
|
3638 |
} |
|
3639 |
||
3640 |
void MacroAssembler::reset_last_Java_frame(Register java_thread, bool clear_fp, bool clear_pc) { |
|
3641 |
// determine java_thread register |
|
3642 |
if (!java_thread->is_valid()) { |
|
3643 |
java_thread = rdi; |
|
3644 |
get_thread(java_thread); |
|
3645 |
} |
|
3646 |
// we must set sp to zero to clear frame |
|
3647 |
movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), NULL_WORD); |
|
3648 |
if (clear_fp) { |
|
3649 |
movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), NULL_WORD); |
|
3650 |
} |
|
3651 |
||
3652 |
if (clear_pc) |
|
3653 |
movptr(Address(java_thread, JavaThread::last_Java_pc_offset()), NULL_WORD); |
|
3654 |
||
3655 |
} |
|
3656 |
||
3657 |
void MacroAssembler::restore_rax(Register tmp) { |
|
3658 |
if (tmp == noreg) pop(rax); |
|
3659 |
else if (tmp != rax) mov(rax, tmp); |
|
3660 |
} |
|
3661 |
||
3662 |
void MacroAssembler::round_to(Register reg, int modulus) { |
|
3663 |
addptr(reg, modulus - 1); |
|
3664 |
andptr(reg, -modulus); |
|
3665 |
} |
|
3666 |
||
3667 |
void MacroAssembler::save_rax(Register tmp) { |
|
3668 |
if (tmp == noreg) push(rax); |
|
3669 |
else if (tmp != rax) mov(tmp, rax); |
|
3670 |
} |
|
3671 |
||
3672 |
// Write serialization page so VM thread can do a pseudo remote membar. |
|
3673 |
// We use the current thread pointer to calculate a thread specific |
|
3674 |
// offset to write to within the page. This minimizes bus traffic |
|
3675 |
// due to cache line collision. |
|
3676 |
void MacroAssembler::serialize_memory(Register thread, Register tmp) { |
|
3677 |
movl(tmp, thread); |
|
3678 |
shrl(tmp, os::get_serialize_page_shift_count()); |
|
3679 |
andl(tmp, (os::vm_page_size() - sizeof(int))); |
|
3680 |
||
3681 |
Address index(noreg, tmp, Address::times_1); |
|
3682 |
ExternalAddress page(os::get_memory_serialize_page()); |
|
3683 |
||
3684 |
// Size of store must match masking code above |
|
3685 |
movl(as_Address(ArrayAddress(page, index)), tmp); |
|
3686 |
} |
|
3687 |
||
3688 |
// Calls to C land |
|
3689 |
// |
|
3690 |
// When entering C land, the rbp, & rsp of the last Java frame have to be recorded |
|
3691 |
// in the (thread-local) JavaThread object. When leaving C land, the last Java fp |
|
3692 |
// has to be reset to 0. This is required to allow proper stack traversal. |
|
3693 |
void MacroAssembler::set_last_Java_frame(Register java_thread, |
|
3694 |
Register last_java_sp, |
|
3695 |
Register last_java_fp, |
|
3696 |
address last_java_pc) { |
|
3697 |
// determine java_thread register |
|
3698 |
if (!java_thread->is_valid()) { |
|
3699 |
java_thread = rdi; |
|
3700 |
get_thread(java_thread); |
|
3701 |
} |
|
3702 |
// determine last_java_sp register |
|
3703 |
if (!last_java_sp->is_valid()) { |
|
3704 |
last_java_sp = rsp; |
|
3705 |
} |
|
3706 |
||
3707 |
// last_java_fp is optional |
|
3708 |
||
3709 |
if (last_java_fp->is_valid()) { |
|
3710 |
movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), last_java_fp); |
|
3711 |
} |
|
3712 |
||
3713 |
// last_java_pc is optional |
|
3714 |
||
3715 |
if (last_java_pc != NULL) { |
|
3716 |
lea(Address(java_thread, |
|
3717 |
JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset()), |
|
3718 |
InternalAddress(last_java_pc)); |
|
3719 |
||
3720 |
} |
|
3721 |
movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), last_java_sp); |
|
3722 |
} |
|
3723 |
||
3724 |
void MacroAssembler::shlptr(Register dst, int imm8) { |
|
3725 |
LP64_ONLY(shlq(dst, imm8)) NOT_LP64(shll(dst, imm8)); |
|
3726 |
} |
|
3727 |
||
3728 |
void MacroAssembler::shrptr(Register dst, int imm8) { |
|
3729 |
LP64_ONLY(shrq(dst, imm8)) NOT_LP64(shrl(dst, imm8)); |
|
3730 |
} |
|
3731 |
||
3732 |
void MacroAssembler::sign_extend_byte(Register reg) { |
|
3733 |
if (LP64_ONLY(true ||) (VM_Version::is_P6() && reg->has_byte_register())) { |
|
3734 |
movsbl(reg, reg); // movsxb |
|
3735 |
} else { |
|
3736 |
shll(reg, 24); |
|
3737 |
sarl(reg, 24); |
|
3738 |
} |
|
3739 |
} |
|
3740 |
||
3741 |
void MacroAssembler::sign_extend_short(Register reg) { |
|
3742 |
if (LP64_ONLY(true ||) VM_Version::is_P6()) { |
|
3743 |
movswl(reg, reg); // movsxw |
|
3744 |
} else { |
|
3745 |
shll(reg, 16); |
|
3746 |
sarl(reg, 16); |
|
3747 |
} |
|
3748 |
} |
|
3749 |
||
3750 |
void MacroAssembler::testl(Register dst, AddressLiteral src) { |
|
3751 |
assert(reachable(src), "Address should be reachable"); |
|
3752 |
testl(dst, as_Address(src)); |
|
3753 |
} |
|
3754 |
||
34203 | 3755 |
void MacroAssembler::pcmpeqb(XMMRegister dst, XMMRegister src) { |
3756 |
int dst_enc = dst->encoding(); |
|
3757 |
int src_enc = src->encoding(); |
|
3758 |
if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) { |
|
3759 |
Assembler::pcmpeqb(dst, src); |
|
3760 |
} else if ((dst_enc < 16) && (src_enc < 16)) { |
|
3761 |
Assembler::pcmpeqb(dst, src); |
|
3762 |
} else if (src_enc < 16) { |
|
3763 |
subptr(rsp, 64); |
|
3764 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
3765 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
3766 |
Assembler::pcmpeqb(xmm0, src); |
|
3767 |
movdqu(dst, xmm0); |
|
3768 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
3769 |
addptr(rsp, 64); |
|
3770 |
} else if (dst_enc < 16) { |
|
3771 |
subptr(rsp, 64); |
|
3772 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
3773 |
evmovdqul(xmm0, src, Assembler::AVX_512bit); |
|
3774 |
Assembler::pcmpeqb(dst, xmm0); |
|
3775 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
3776 |
addptr(rsp, 64); |
|
3777 |
} else { |
|
3778 |
subptr(rsp, 64); |
|
3779 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
3780 |
subptr(rsp, 64); |
|
3781 |
evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit); |
|
3782 |
movdqu(xmm0, src); |
|
3783 |
movdqu(xmm1, dst); |
|
3784 |
Assembler::pcmpeqb(xmm1, xmm0); |
|
3785 |
movdqu(dst, xmm1); |
|
3786 |
evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit); |
|
3787 |
addptr(rsp, 64); |
|
3788 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
3789 |
addptr(rsp, 64); |
|
3790 |
} |
|
3791 |
} |
|
3792 |
||
3793 |
void MacroAssembler::pcmpeqw(XMMRegister dst, XMMRegister src) { |
|
3794 |
int dst_enc = dst->encoding(); |
|
3795 |
int src_enc = src->encoding(); |
|
3796 |
if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) { |
|
3797 |
Assembler::pcmpeqw(dst, src); |
|
3798 |
} else if ((dst_enc < 16) && (src_enc < 16)) { |
|
3799 |
Assembler::pcmpeqw(dst, src); |
|
3800 |
} else if (src_enc < 16) { |
|
3801 |
subptr(rsp, 64); |
|
3802 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
3803 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
3804 |
Assembler::pcmpeqw(xmm0, src); |
|
3805 |
movdqu(dst, xmm0); |
|
3806 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
3807 |
addptr(rsp, 64); |
|
3808 |
} else if (dst_enc < 16) { |
|
3809 |
subptr(rsp, 64); |
|
3810 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
3811 |
evmovdqul(xmm0, src, Assembler::AVX_512bit); |
|
3812 |
Assembler::pcmpeqw(dst, xmm0); |
|
3813 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
3814 |
addptr(rsp, 64); |
|
3815 |
} else { |
|
3816 |
subptr(rsp, 64); |
|
3817 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
3818 |
subptr(rsp, 64); |
|
3819 |
evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit); |
|
3820 |
movdqu(xmm0, src); |
|
3821 |
movdqu(xmm1, dst); |
|
3822 |
Assembler::pcmpeqw(xmm1, xmm0); |
|
3823 |
movdqu(dst, xmm1); |
|
3824 |
evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit); |
|
3825 |
addptr(rsp, 64); |
|
3826 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
3827 |
addptr(rsp, 64); |
|
3828 |
} |
|
3829 |
} |
|
3830 |
||
3831 |
void MacroAssembler::pcmpestri(XMMRegister dst, Address src, int imm8) { |
|
3832 |
int dst_enc = dst->encoding(); |
|
3833 |
if (dst_enc < 16) { |
|
3834 |
Assembler::pcmpestri(dst, src, imm8); |
|
3835 |
} else { |
|
3836 |
subptr(rsp, 64); |
|
3837 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
3838 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
3839 |
Assembler::pcmpestri(xmm0, src, imm8); |
|
3840 |
movdqu(dst, xmm0); |
|
3841 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
3842 |
addptr(rsp, 64); |
|
3843 |
} |
|
3844 |
} |
|
3845 |
||
3846 |
void MacroAssembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) { |
|
3847 |
int dst_enc = dst->encoding(); |
|
3848 |
int src_enc = src->encoding(); |
|
3849 |
if ((dst_enc < 16) && (src_enc < 16)) { |
|
3850 |
Assembler::pcmpestri(dst, src, imm8); |
|
3851 |
} else if (src_enc < 16) { |
|
3852 |
subptr(rsp, 64); |
|
3853 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
3854 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
3855 |
Assembler::pcmpestri(xmm0, src, imm8); |
|
3856 |
movdqu(dst, xmm0); |
|
3857 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
3858 |
addptr(rsp, 64); |
|
3859 |
} else if (dst_enc < 16) { |
|
3860 |
subptr(rsp, 64); |
|
3861 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
3862 |
evmovdqul(xmm0, src, Assembler::AVX_512bit); |
|
3863 |
Assembler::pcmpestri(dst, xmm0, imm8); |
|
3864 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
3865 |
addptr(rsp, 64); |
|
3866 |
} else { |
|
3867 |
subptr(rsp, 64); |
|
3868 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
3869 |
subptr(rsp, 64); |
|
3870 |
evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit); |
|
3871 |
movdqu(xmm0, src); |
|
3872 |
movdqu(xmm1, dst); |
|
3873 |
Assembler::pcmpestri(xmm1, xmm0, imm8); |
|
3874 |
movdqu(dst, xmm1); |
|
3875 |
evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit); |
|
3876 |
addptr(rsp, 64); |
|
3877 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
3878 |
addptr(rsp, 64); |
|
3879 |
} |
|
3880 |
} |
|
3881 |
||
3882 |
void MacroAssembler::pmovzxbw(XMMRegister dst, XMMRegister src) { |
|
3883 |
int dst_enc = dst->encoding(); |
|
3884 |
int src_enc = src->encoding(); |
|
3885 |
if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) { |
|
3886 |
Assembler::pmovzxbw(dst, src); |
|
3887 |
} else if ((dst_enc < 16) && (src_enc < 16)) { |
|
3888 |
Assembler::pmovzxbw(dst, src); |
|
3889 |
} else if (src_enc < 16) { |
|
3890 |
subptr(rsp, 64); |
|
3891 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
3892 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
3893 |
Assembler::pmovzxbw(xmm0, src); |
|
3894 |
movdqu(dst, xmm0); |
|
3895 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
3896 |
addptr(rsp, 64); |
|
3897 |
} else if (dst_enc < 16) { |
|
3898 |
subptr(rsp, 64); |
|
3899 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
3900 |
evmovdqul(xmm0, src, Assembler::AVX_512bit); |
|
3901 |
Assembler::pmovzxbw(dst, xmm0); |
|
3902 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
3903 |
addptr(rsp, 64); |
|
3904 |
} else { |
|
3905 |
subptr(rsp, 64); |
|
3906 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
3907 |
subptr(rsp, 64); |
|
3908 |
evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit); |
|
3909 |
movdqu(xmm0, src); |
|
3910 |
movdqu(xmm1, dst); |
|
3911 |
Assembler::pmovzxbw(xmm1, xmm0); |
|
3912 |
movdqu(dst, xmm1); |
|
3913 |
evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit); |
|
3914 |
addptr(rsp, 64); |
|
3915 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
3916 |
addptr(rsp, 64); |
|
3917 |
} |
|
3918 |
} |
|
3919 |
||
3920 |
void MacroAssembler::pmovzxbw(XMMRegister dst, Address src) { |
|
3921 |
int dst_enc = dst->encoding(); |
|
3922 |
if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) { |
|
3923 |
Assembler::pmovzxbw(dst, src); |
|
3924 |
} else if (dst_enc < 16) { |
|
3925 |
Assembler::pmovzxbw(dst, src); |
|
3926 |
} else { |
|
3927 |
subptr(rsp, 64); |
|
3928 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
3929 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
3930 |
Assembler::pmovzxbw(xmm0, src); |
|
3931 |
movdqu(dst, xmm0); |
|
3932 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
3933 |
addptr(rsp, 64); |
|
3934 |
} |
|
3935 |
} |
|
3936 |
||
3937 |
void MacroAssembler::pmovmskb(Register dst, XMMRegister src) { |
|
3938 |
int src_enc = src->encoding(); |
|
3939 |
if (src_enc < 16) { |
|
3940 |
Assembler::pmovmskb(dst, src); |
|
3941 |
} else { |
|
3942 |
subptr(rsp, 64); |
|
3943 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
3944 |
evmovdqul(xmm0, src, Assembler::AVX_512bit); |
|
3945 |
Assembler::pmovmskb(dst, xmm0); |
|
3946 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
3947 |
addptr(rsp, 64); |
|
3948 |
} |
|
3949 |
} |
|
3950 |
||
3951 |
void MacroAssembler::ptest(XMMRegister dst, XMMRegister src) { |
|
3952 |
int dst_enc = dst->encoding(); |
|
3953 |
int src_enc = src->encoding(); |
|
3954 |
if ((dst_enc < 16) && (src_enc < 16)) { |
|
3955 |
Assembler::ptest(dst, src); |
|
3956 |
} else if (src_enc < 16) { |
|
3957 |
subptr(rsp, 64); |
|
3958 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
3959 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
3960 |
Assembler::ptest(xmm0, src); |
|
3961 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
3962 |
addptr(rsp, 64); |
|
3963 |
} else if (dst_enc < 16) { |
|
3964 |
subptr(rsp, 64); |
|
3965 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
3966 |
evmovdqul(xmm0, src, Assembler::AVX_512bit); |
|
3967 |
Assembler::ptest(dst, xmm0); |
|
3968 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
3969 |
addptr(rsp, 64); |
|
3970 |
} else { |
|
3971 |
subptr(rsp, 64); |
|
3972 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
3973 |
subptr(rsp, 64); |
|
3974 |
evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit); |
|
3975 |
movdqu(xmm0, src); |
|
3976 |
movdqu(xmm1, dst); |
|
3977 |
Assembler::ptest(xmm1, xmm0); |
|
3978 |
evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit); |
|
3979 |
addptr(rsp, 64); |
|
3980 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
3981 |
addptr(rsp, 64); |
|
3982 |
} |
|
3983 |
} |
|
3984 |
||
14626 | 3985 |
void MacroAssembler::sqrtsd(XMMRegister dst, AddressLiteral src) { |
3986 |
if (reachable(src)) { |
|
3987 |
Assembler::sqrtsd(dst, as_Address(src)); |
|
3988 |
} else { |
|
3989 |
lea(rscratch1, src); |
|
3990 |
Assembler::sqrtsd(dst, Address(rscratch1, 0)); |
|
3991 |
} |
|
3992 |
} |
|
3993 |
||
3994 |
void MacroAssembler::sqrtss(XMMRegister dst, AddressLiteral src) { |
|
3995 |
if (reachable(src)) { |
|
3996 |
Assembler::sqrtss(dst, as_Address(src)); |
|
3997 |
} else { |
|
3998 |
lea(rscratch1, src); |
|
3999 |
Assembler::sqrtss(dst, Address(rscratch1, 0)); |
|
4000 |
} |
|
4001 |
} |
|
4002 |
||
4003 |
void MacroAssembler::subsd(XMMRegister dst, AddressLiteral src) { |
|
4004 |
if (reachable(src)) { |
|
4005 |
Assembler::subsd(dst, as_Address(src)); |
|
4006 |
} else { |
|
4007 |
lea(rscratch1, src); |
|
4008 |
Assembler::subsd(dst, Address(rscratch1, 0)); |
|
4009 |
} |
|
4010 |
} |
|
4011 |
||
4012 |
void MacroAssembler::subss(XMMRegister dst, AddressLiteral src) { |
|
4013 |
if (reachable(src)) { |
|
4014 |
Assembler::subss(dst, as_Address(src)); |
|
4015 |
} else { |
|
4016 |
lea(rscratch1, src); |
|
4017 |
Assembler::subss(dst, Address(rscratch1, 0)); |
|
4018 |
} |
|
4019 |
} |
|
4020 |
||
4021 |
void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src) { |
|
4022 |
if (reachable(src)) { |
|
4023 |
Assembler::ucomisd(dst, as_Address(src)); |
|
4024 |
} else { |
|
4025 |
lea(rscratch1, src); |
|
4026 |
Assembler::ucomisd(dst, Address(rscratch1, 0)); |
|
4027 |
} |
|
4028 |
} |
|
4029 |
||
4030 |
void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src) { |
|
4031 |
if (reachable(src)) { |
|
4032 |
Assembler::ucomiss(dst, as_Address(src)); |
|
4033 |
} else { |
|
4034 |
lea(rscratch1, src); |
|
4035 |
Assembler::ucomiss(dst, Address(rscratch1, 0)); |
|
4036 |
} |
|
4037 |
} |
|
4038 |
||
4039 |
void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src) { |
|
4040 |
// Used in sign-bit flipping with aligned address. |
|
4041 |
assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); |
|
4042 |
if (reachable(src)) { |
|
4043 |
Assembler::xorpd(dst, as_Address(src)); |
|
4044 |
} else { |
|
4045 |
lea(rscratch1, src); |
|
4046 |
Assembler::xorpd(dst, Address(rscratch1, 0)); |
|
4047 |
} |
|
4048 |
} |
|
4049 |
||
34162 | 4050 |
void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) { |
4051 |
if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) { |
|
4052 |
Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit); |
|
4053 |
} |
|
4054 |
else { |
|
4055 |
Assembler::xorpd(dst, src); |
|
4056 |
} |
|
4057 |
} |
|
4058 |
||
4059 |
void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) { |
|
4060 |
if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) { |
|
4061 |
Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit); |
|
4062 |
} else { |
|
4063 |
Assembler::xorps(dst, src); |
|
4064 |
} |
|
4065 |
} |
|
4066 |
||
14626 | 4067 |
void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src) { |
4068 |
// Used in sign-bit flipping with aligned address. |
|
4069 |
assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); |
|
4070 |
if (reachable(src)) { |
|
4071 |
Assembler::xorps(dst, as_Address(src)); |
|
4072 |
} else { |
|
4073 |
lea(rscratch1, src); |
|
4074 |
Assembler::xorps(dst, Address(rscratch1, 0)); |
|
4075 |
} |
|
4076 |
} |
|
4077 |
||
4078 |
void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src) { |
|
4079 |
// Used in sign-bit flipping with aligned address. |
|
14834 | 4080 |
bool aligned_adr = (((intptr_t)src.target() & 15) == 0); |
4081 |
assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes"); |
|
14626 | 4082 |
if (reachable(src)) { |
4083 |
Assembler::pshufb(dst, as_Address(src)); |
|
4084 |
} else { |
|
4085 |
lea(rscratch1, src); |
|
4086 |
Assembler::pshufb(dst, Address(rscratch1, 0)); |
|
4087 |
} |
|
4088 |
} |
|
4089 |
||
4090 |
// AVX 3-operands instructions |
|
4091 |
||
4092 |
void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) { |
|
4093 |
if (reachable(src)) { |
|
4094 |
vaddsd(dst, nds, as_Address(src)); |
|
4095 |
} else { |
|
4096 |
lea(rscratch1, src); |
|
4097 |
vaddsd(dst, nds, Address(rscratch1, 0)); |
|
4098 |
} |
|
4099 |
} |
|
4100 |
||
4101 |
void MacroAssembler::vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src) { |
|
4102 |
if (reachable(src)) { |
|
4103 |
vaddss(dst, nds, as_Address(src)); |
|
4104 |
} else { |
|
4105 |
lea(rscratch1, src); |
|
4106 |
vaddss(dst, nds, Address(rscratch1, 0)); |
|
4107 |
} |
|
4108 |
} |
|
4109 |
||
34162 | 4110 |
void MacroAssembler::vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len) { |
4111 |
int dst_enc = dst->encoding(); |
|
4112 |
int nds_enc = nds->encoding(); |
|
4113 |
int src_enc = src->encoding(); |
|
4114 |
if ((dst_enc < 16) && (nds_enc < 16)) { |
|
4115 |
vandps(dst, nds, negate_field, vector_len); |
|
4116 |
} else if ((src_enc < 16) && (dst_enc < 16)) { |
|
4117 |
movss(src, nds); |
|
4118 |
vandps(dst, src, negate_field, vector_len); |
|
4119 |
} else if (src_enc < 16) { |
|
4120 |
movss(src, nds); |
|
4121 |
vandps(src, src, negate_field, vector_len); |
|
4122 |
movss(dst, src); |
|
4123 |
} else if (dst_enc < 16) { |
|
4124 |
movdqu(src, xmm0); |
|
4125 |
movss(xmm0, nds); |
|
4126 |
vandps(dst, xmm0, negate_field, vector_len); |
|
4127 |
movdqu(xmm0, src); |
|
4128 |
} else if (nds_enc < 16) { |
|
4129 |
movdqu(src, xmm0); |
|
4130 |
vandps(xmm0, nds, negate_field, vector_len); |
|
4131 |
movss(dst, xmm0); |
|
4132 |
movdqu(xmm0, src); |
|
4133 |
} else { |
|
4134 |
movdqu(src, xmm0); |
|
4135 |
movss(xmm0, nds); |
|
4136 |
vandps(xmm0, xmm0, negate_field, vector_len); |
|
4137 |
movss(dst, xmm0); |
|
4138 |
movdqu(xmm0, src); |
|
4139 |
} |
|
4140 |
} |
|
4141 |
||
4142 |
void MacroAssembler::vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len) { |
|
4143 |
int dst_enc = dst->encoding(); |
|
4144 |
int nds_enc = nds->encoding(); |
|
4145 |
int src_enc = src->encoding(); |
|
4146 |
if ((dst_enc < 16) && (nds_enc < 16)) { |
|
4147 |
vandpd(dst, nds, negate_field, vector_len); |
|
4148 |
} else if ((src_enc < 16) && (dst_enc < 16)) { |
|
4149 |
movsd(src, nds); |
|
4150 |
vandpd(dst, src, negate_field, vector_len); |
|
4151 |
} else if (src_enc < 16) { |
|
4152 |
movsd(src, nds); |
|
4153 |
vandpd(src, src, negate_field, vector_len); |
|
4154 |
movsd(dst, src); |
|
4155 |
} else if (dst_enc < 16) { |
|
4156 |
movdqu(src, xmm0); |
|
4157 |
movsd(xmm0, nds); |
|
4158 |
vandpd(dst, xmm0, negate_field, vector_len); |
|
4159 |
movdqu(xmm0, src); |
|
4160 |
} else if (nds_enc < 16) { |
|
4161 |
movdqu(src, xmm0); |
|
4162 |
vandpd(xmm0, nds, negate_field, vector_len); |
|
4163 |
movsd(dst, xmm0); |
|
4164 |
movdqu(xmm0, src); |
|
4165 |
} else { |
|
4166 |
movdqu(src, xmm0); |
|
4167 |
movsd(xmm0, nds); |
|
4168 |
vandpd(xmm0, xmm0, negate_field, vector_len); |
|
4169 |
movsd(dst, xmm0); |
|
4170 |
movdqu(xmm0, src); |
|
4171 |
} |
|
4172 |
} |
|
4173 |
||
4174 |
void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { |
|
4175 |
int dst_enc = dst->encoding(); |
|
4176 |
int nds_enc = nds->encoding(); |
|
4177 |
int src_enc = src->encoding(); |
|
4178 |
if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) { |
|
4179 |
Assembler::vpaddb(dst, nds, src, vector_len); |
|
4180 |
} else if ((dst_enc < 16) && (src_enc < 16)) { |
|
4181 |
Assembler::vpaddb(dst, dst, src, vector_len); |
|
4182 |
} else if ((dst_enc < 16) && (nds_enc < 16)) { |
|
4183 |
// use nds as scratch for src |
|
4184 |
evmovdqul(nds, src, Assembler::AVX_512bit); |
|
4185 |
Assembler::vpaddb(dst, dst, nds, vector_len); |
|
4186 |
} else if ((src_enc < 16) && (nds_enc < 16)) { |
|
4187 |
// use nds as scratch for dst |
|
4188 |
evmovdqul(nds, dst, Assembler::AVX_512bit); |
|
4189 |
Assembler::vpaddb(nds, nds, src, vector_len); |
|
4190 |
evmovdqul(dst, nds, Assembler::AVX_512bit); |
|
4191 |
} else if (dst_enc < 16) { |
|
4192 |
// use nds as scatch for xmm0 to hold src |
|
4193 |
evmovdqul(nds, xmm0, Assembler::AVX_512bit); |
|
4194 |
evmovdqul(xmm0, src, Assembler::AVX_512bit); |
|
4195 |
Assembler::vpaddb(dst, dst, xmm0, vector_len); |
|
4196 |
evmovdqul(xmm0, nds, Assembler::AVX_512bit); |
|
4197 |
} else { |
|
4198 |
// worse case scenario, all regs are in the upper bank |
|
4199 |
subptr(rsp, 64); |
|
4200 |
evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit); |
|
4201 |
evmovdqul(nds, xmm0, Assembler::AVX_512bit); |
|
4202 |
evmovdqul(xmm1, src, Assembler::AVX_512bit); |
|
4203 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4204 |
Assembler::vpaddb(xmm0, xmm0, xmm1, vector_len); |
|
4205 |
evmovdqul(dst, xmm0, Assembler::AVX_512bit); |
|
4206 |
evmovdqul(xmm0, nds, Assembler::AVX_512bit); |
|
4207 |
evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit); |
|
4208 |
addptr(rsp, 64); |
|
4209 |
} |
|
4210 |
} |
|
4211 |
||
4212 |
void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { |
|
4213 |
int dst_enc = dst->encoding(); |
|
4214 |
int nds_enc = nds->encoding(); |
|
4215 |
if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) { |
|
4216 |
Assembler::vpaddb(dst, nds, src, vector_len); |
|
4217 |
} else if (dst_enc < 16) { |
|
4218 |
Assembler::vpaddb(dst, dst, src, vector_len); |
|
4219 |
} else if (nds_enc < 16) { |
|
4220 |
// implies dst_enc in upper bank with src as scratch |
|
4221 |
evmovdqul(nds, dst, Assembler::AVX_512bit); |
|
4222 |
Assembler::vpaddb(nds, nds, src, vector_len); |
|
4223 |
evmovdqul(dst, nds, Assembler::AVX_512bit); |
|
4224 |
} else { |
|
4225 |
// worse case scenario, all regs in upper bank |
|
4226 |
evmovdqul(nds, xmm0, Assembler::AVX_512bit); |
|
4227 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4228 |
Assembler::vpaddb(xmm0, xmm0, src, vector_len); |
|
4229 |
evmovdqul(xmm0, nds, Assembler::AVX_512bit); |
|
4230 |
} |
|
4231 |
} |
|
4232 |
||
4233 |
void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { |
|
4234 |
int dst_enc = dst->encoding(); |
|
4235 |
int nds_enc = nds->encoding(); |
|
4236 |
int src_enc = src->encoding(); |
|
4237 |
if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) { |
|
4238 |
Assembler::vpaddw(dst, nds, src, vector_len); |
|
4239 |
} else if ((dst_enc < 16) && (src_enc < 16)) { |
|
4240 |
Assembler::vpaddw(dst, dst, src, vector_len); |
|
4241 |
} else if ((dst_enc < 16) && (nds_enc < 16)) { |
|
4242 |
// use nds as scratch for src |
|
4243 |
evmovdqul(nds, src, Assembler::AVX_512bit); |
|
4244 |
Assembler::vpaddw(dst, dst, nds, vector_len); |
|
4245 |
} else if ((src_enc < 16) && (nds_enc < 16)) { |
|
4246 |
// use nds as scratch for dst |
|
4247 |
evmovdqul(nds, dst, Assembler::AVX_512bit); |
|
4248 |
Assembler::vpaddw(nds, nds, src, vector_len); |
|
4249 |
evmovdqul(dst, nds, Assembler::AVX_512bit); |
|
4250 |
} else if (dst_enc < 16) { |
|
4251 |
// use nds as scatch for xmm0 to hold src |
|
4252 |
evmovdqul(nds, xmm0, Assembler::AVX_512bit); |
|
4253 |
evmovdqul(xmm0, src, Assembler::AVX_512bit); |
|
4254 |
Assembler::vpaddw(dst, dst, xmm0, vector_len); |
|
4255 |
evmovdqul(xmm0, nds, Assembler::AVX_512bit); |
|
4256 |
} else { |
|
4257 |
// worse case scenario, all regs are in the upper bank |
|
4258 |
subptr(rsp, 64); |
|
4259 |
evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit); |
|
4260 |
evmovdqul(nds, xmm0, Assembler::AVX_512bit); |
|
4261 |
evmovdqul(xmm1, src, Assembler::AVX_512bit); |
|
4262 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4263 |
Assembler::vpaddw(xmm0, xmm0, xmm1, vector_len); |
|
4264 |
evmovdqul(dst, xmm0, Assembler::AVX_512bit); |
|
4265 |
evmovdqul(xmm0, nds, Assembler::AVX_512bit); |
|
4266 |
evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit); |
|
4267 |
addptr(rsp, 64); |
|
4268 |
} |
|
4269 |
} |
|
4270 |
||
4271 |
void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { |
|
4272 |
int dst_enc = dst->encoding(); |
|
4273 |
int nds_enc = nds->encoding(); |
|
4274 |
if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) { |
|
4275 |
Assembler::vpaddw(dst, nds, src, vector_len); |
|
4276 |
} else if (dst_enc < 16) { |
|
4277 |
Assembler::vpaddw(dst, dst, src, vector_len); |
|
4278 |
} else if (nds_enc < 16) { |
|
4279 |
// implies dst_enc in upper bank with src as scratch |
|
4280 |
evmovdqul(nds, dst, Assembler::AVX_512bit); |
|
4281 |
Assembler::vpaddw(nds, nds, src, vector_len); |
|
4282 |
evmovdqul(dst, nds, Assembler::AVX_512bit); |
|
4283 |
} else { |
|
4284 |
// worse case scenario, all regs in upper bank |
|
4285 |
evmovdqul(nds, xmm0, Assembler::AVX_512bit); |
|
4286 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4287 |
Assembler::vpaddw(xmm0, xmm0, src, vector_len); |
|
4288 |
evmovdqul(xmm0, nds, Assembler::AVX_512bit); |
|
4289 |
} |
|
4290 |
} |
|
4291 |
||
34203 | 4292 |
void MacroAssembler::vpbroadcastw(XMMRegister dst, XMMRegister src) { |
4293 |
int dst_enc = dst->encoding(); |
|
4294 |
int src_enc = src->encoding(); |
|
4295 |
if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) { |
|
4296 |
Assembler::vpbroadcastw(dst, src); |
|
4297 |
} else if ((dst_enc < 16) && (src_enc < 16)) { |
|
4298 |
Assembler::vpbroadcastw(dst, src); |
|
4299 |
} else if (src_enc < 16) { |
|
4300 |
subptr(rsp, 64); |
|
4301 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
4302 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4303 |
Assembler::vpbroadcastw(xmm0, src); |
|
4304 |
movdqu(dst, xmm0); |
|
4305 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
4306 |
addptr(rsp, 64); |
|
4307 |
} else if (dst_enc < 16) { |
|
4308 |
subptr(rsp, 64); |
|
4309 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
4310 |
evmovdqul(xmm0, src, Assembler::AVX_512bit); |
|
4311 |
Assembler::vpbroadcastw(dst, xmm0); |
|
4312 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
4313 |
addptr(rsp, 64); |
|
4314 |
} else { |
|
4315 |
subptr(rsp, 64); |
|
4316 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
4317 |
subptr(rsp, 64); |
|
4318 |
evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit); |
|
4319 |
movdqu(xmm0, src); |
|
4320 |
movdqu(xmm1, dst); |
|
4321 |
Assembler::vpbroadcastw(xmm1, xmm0); |
|
4322 |
movdqu(dst, xmm1); |
|
4323 |
evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit); |
|
4324 |
addptr(rsp, 64); |
|
4325 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
4326 |
addptr(rsp, 64); |
|
4327 |
} |
|
4328 |
} |
|
4329 |
||
4330 |
void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { |
|
4331 |
int dst_enc = dst->encoding(); |
|
4332 |
int nds_enc = nds->encoding(); |
|
4333 |
int src_enc = src->encoding(); |
|
4334 |
assert(dst_enc == nds_enc, ""); |
|
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
4335 |
if ((dst_enc < 16) && (src_enc < 16)) { |
34203 | 4336 |
Assembler::vpcmpeqb(dst, nds, src, vector_len); |
4337 |
} else if (src_enc < 16) { |
|
4338 |
subptr(rsp, 64); |
|
4339 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
4340 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4341 |
Assembler::vpcmpeqb(xmm0, xmm0, src, vector_len); |
|
4342 |
movdqu(dst, xmm0); |
|
4343 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
4344 |
addptr(rsp, 64); |
|
4345 |
} else if (dst_enc < 16) { |
|
4346 |
subptr(rsp, 64); |
|
4347 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
4348 |
evmovdqul(xmm0, src, Assembler::AVX_512bit); |
|
4349 |
Assembler::vpcmpeqb(dst, dst, xmm0, vector_len); |
|
4350 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
4351 |
addptr(rsp, 64); |
|
4352 |
} else { |
|
4353 |
subptr(rsp, 64); |
|
4354 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
4355 |
subptr(rsp, 64); |
|
4356 |
evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit); |
|
4357 |
movdqu(xmm0, src); |
|
4358 |
movdqu(xmm1, dst); |
|
4359 |
Assembler::vpcmpeqb(xmm1, xmm1, xmm0, vector_len); |
|
4360 |
movdqu(dst, xmm1); |
|
4361 |
evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit); |
|
4362 |
addptr(rsp, 64); |
|
4363 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
4364 |
addptr(rsp, 64); |
|
4365 |
} |
|
4366 |
} |
|
4367 |
||
4368 |
void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { |
|
4369 |
int dst_enc = dst->encoding(); |
|
4370 |
int nds_enc = nds->encoding(); |
|
4371 |
int src_enc = src->encoding(); |
|
4372 |
assert(dst_enc == nds_enc, ""); |
|
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
4373 |
if ((dst_enc < 16) && (src_enc < 16)) { |
34203 | 4374 |
Assembler::vpcmpeqw(dst, nds, src, vector_len); |
4375 |
} else if (src_enc < 16) { |
|
4376 |
subptr(rsp, 64); |
|
4377 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
4378 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4379 |
Assembler::vpcmpeqw(xmm0, xmm0, src, vector_len); |
|
4380 |
movdqu(dst, xmm0); |
|
4381 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
4382 |
addptr(rsp, 64); |
|
4383 |
} else if (dst_enc < 16) { |
|
4384 |
subptr(rsp, 64); |
|
4385 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
4386 |
evmovdqul(xmm0, src, Assembler::AVX_512bit); |
|
4387 |
Assembler::vpcmpeqw(dst, dst, xmm0, vector_len); |
|
4388 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
4389 |
addptr(rsp, 64); |
|
4390 |
} else { |
|
4391 |
subptr(rsp, 64); |
|
4392 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
4393 |
subptr(rsp, 64); |
|
4394 |
evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit); |
|
4395 |
movdqu(xmm0, src); |
|
4396 |
movdqu(xmm1, dst); |
|
4397 |
Assembler::vpcmpeqw(xmm1, xmm1, xmm0, vector_len); |
|
4398 |
movdqu(dst, xmm1); |
|
4399 |
evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit); |
|
4400 |
addptr(rsp, 64); |
|
4401 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
4402 |
addptr(rsp, 64); |
|
4403 |
} |
|
4404 |
} |
|
4405 |
||
4406 |
void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) { |
|
4407 |
int dst_enc = dst->encoding(); |
|
4408 |
if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) { |
|
4409 |
Assembler::vpmovzxbw(dst, src, vector_len); |
|
4410 |
} else if (dst_enc < 16) { |
|
4411 |
Assembler::vpmovzxbw(dst, src, vector_len); |
|
4412 |
} else { |
|
4413 |
subptr(rsp, 64); |
|
4414 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
4415 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4416 |
Assembler::vpmovzxbw(xmm0, src, vector_len); |
|
4417 |
movdqu(dst, xmm0); |
|
4418 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
4419 |
addptr(rsp, 64); |
|
4420 |
} |
|
4421 |
} |
|
4422 |
||
4423 |
void MacroAssembler::vpmovmskb(Register dst, XMMRegister src) { |
|
4424 |
int src_enc = src->encoding(); |
|
4425 |
if (src_enc < 16) { |
|
4426 |
Assembler::vpmovmskb(dst, src); |
|
4427 |
} else { |
|
4428 |
subptr(rsp, 64); |
|
4429 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
4430 |
evmovdqul(xmm0, src, Assembler::AVX_512bit); |
|
4431 |
Assembler::vpmovmskb(dst, xmm0); |
|
4432 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
4433 |
addptr(rsp, 64); |
|
4434 |
} |
|
4435 |
} |
|
4436 |
||
4437 |
void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { |
|
4438 |
int dst_enc = dst->encoding(); |
|
4439 |
int nds_enc = nds->encoding(); |
|
4440 |
int src_enc = src->encoding(); |
|
4441 |
if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) { |
|
4442 |
Assembler::vpmullw(dst, nds, src, vector_len); |
|
4443 |
} else if ((dst_enc < 16) && (src_enc < 16)) { |
|
4444 |
Assembler::vpmullw(dst, dst, src, vector_len); |
|
4445 |
} else if ((dst_enc < 16) && (nds_enc < 16)) { |
|
4446 |
// use nds as scratch for src |
|
4447 |
evmovdqul(nds, src, Assembler::AVX_512bit); |
|
4448 |
Assembler::vpmullw(dst, dst, nds, vector_len); |
|
4449 |
} else if ((src_enc < 16) && (nds_enc < 16)) { |
|
4450 |
// use nds as scratch for dst |
|
4451 |
evmovdqul(nds, dst, Assembler::AVX_512bit); |
|
4452 |
Assembler::vpmullw(nds, nds, src, vector_len); |
|
4453 |
evmovdqul(dst, nds, Assembler::AVX_512bit); |
|
4454 |
} else if (dst_enc < 16) { |
|
4455 |
// use nds as scatch for xmm0 to hold src |
|
4456 |
evmovdqul(nds, xmm0, Assembler::AVX_512bit); |
|
4457 |
evmovdqul(xmm0, src, Assembler::AVX_512bit); |
|
4458 |
Assembler::vpmullw(dst, dst, xmm0, vector_len); |
|
4459 |
evmovdqul(xmm0, nds, Assembler::AVX_512bit); |
|
4460 |
} else { |
|
4461 |
// worse case scenario, all regs are in the upper bank |
|
4462 |
subptr(rsp, 64); |
|
4463 |
evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit); |
|
4464 |
evmovdqul(nds, xmm0, Assembler::AVX_512bit); |
|
4465 |
evmovdqul(xmm1, src, Assembler::AVX_512bit); |
|
4466 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4467 |
Assembler::vpmullw(xmm0, xmm0, xmm1, vector_len); |
|
4468 |
evmovdqul(dst, xmm0, Assembler::AVX_512bit); |
|
4469 |
evmovdqul(xmm0, nds, Assembler::AVX_512bit); |
|
4470 |
evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit); |
|
4471 |
addptr(rsp, 64); |
|
4472 |
} |
|
4473 |
} |
|
4474 |
||
4475 |
void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { |
|
4476 |
int dst_enc = dst->encoding(); |
|
4477 |
int nds_enc = nds->encoding(); |
|
4478 |
if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) { |
|
4479 |
Assembler::vpmullw(dst, nds, src, vector_len); |
|
4480 |
} else if (dst_enc < 16) { |
|
4481 |
Assembler::vpmullw(dst, dst, src, vector_len); |
|
4482 |
} else if (nds_enc < 16) { |
|
4483 |
// implies dst_enc in upper bank with src as scratch |
|
4484 |
evmovdqul(nds, dst, Assembler::AVX_512bit); |
|
4485 |
Assembler::vpmullw(nds, nds, src, vector_len); |
|
4486 |
evmovdqul(dst, nds, Assembler::AVX_512bit); |
|
4487 |
} else { |
|
4488 |
// worse case scenario, all regs in upper bank |
|
4489 |
evmovdqul(nds, xmm0, Assembler::AVX_512bit); |
|
4490 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4491 |
Assembler::vpmullw(xmm0, xmm0, src, vector_len); |
|
4492 |
evmovdqul(xmm0, nds, Assembler::AVX_512bit); |
|
4493 |
} |
|
4494 |
} |
|
4495 |
||
34162 | 4496 |
void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { |
4497 |
int dst_enc = dst->encoding(); |
|
4498 |
int nds_enc = nds->encoding(); |
|
4499 |
int src_enc = src->encoding(); |
|
4500 |
if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) { |
|
4501 |
Assembler::vpsubb(dst, nds, src, vector_len); |
|
4502 |
} else if ((dst_enc < 16) && (src_enc < 16)) { |
|
4503 |
Assembler::vpsubb(dst, dst, src, vector_len); |
|
4504 |
} else if ((dst_enc < 16) && (nds_enc < 16)) { |
|
4505 |
// use nds as scratch for src |
|
4506 |
evmovdqul(nds, src, Assembler::AVX_512bit); |
|
4507 |
Assembler::vpsubb(dst, dst, nds, vector_len); |
|
4508 |
} else if ((src_enc < 16) && (nds_enc < 16)) { |
|
4509 |
// use nds as scratch for dst |
|
4510 |
evmovdqul(nds, dst, Assembler::AVX_512bit); |
|
4511 |
Assembler::vpsubb(nds, nds, src, vector_len); |
|
4512 |
evmovdqul(dst, nds, Assembler::AVX_512bit); |
|
4513 |
} else if (dst_enc < 16) { |
|
4514 |
// use nds as scatch for xmm0 to hold src |
|
4515 |
evmovdqul(nds, xmm0, Assembler::AVX_512bit); |
|
4516 |
evmovdqul(xmm0, src, Assembler::AVX_512bit); |
|
4517 |
Assembler::vpsubb(dst, dst, xmm0, vector_len); |
|
4518 |
evmovdqul(xmm0, nds, Assembler::AVX_512bit); |
|
4519 |
} else { |
|
4520 |
// worse case scenario, all regs are in the upper bank |
|
4521 |
subptr(rsp, 64); |
|
4522 |
evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit); |
|
4523 |
evmovdqul(nds, xmm0, Assembler::AVX_512bit); |
|
4524 |
evmovdqul(xmm1, src, Assembler::AVX_512bit); |
|
4525 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4526 |
Assembler::vpsubb(xmm0, xmm0, xmm1, vector_len); |
|
4527 |
evmovdqul(dst, xmm0, Assembler::AVX_512bit); |
|
4528 |
evmovdqul(xmm0, nds, Assembler::AVX_512bit); |
|
4529 |
evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit); |
|
4530 |
addptr(rsp, 64); |
|
4531 |
} |
|
4532 |
} |
|
4533 |
||
4534 |
void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { |
|
4535 |
int dst_enc = dst->encoding(); |
|
4536 |
int nds_enc = nds->encoding(); |
|
4537 |
if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) { |
|
4538 |
Assembler::vpsubb(dst, nds, src, vector_len); |
|
4539 |
} else if (dst_enc < 16) { |
|
4540 |
Assembler::vpsubb(dst, dst, src, vector_len); |
|
4541 |
} else if (nds_enc < 16) { |
|
4542 |
// implies dst_enc in upper bank with src as scratch |
|
4543 |
evmovdqul(nds, dst, Assembler::AVX_512bit); |
|
4544 |
Assembler::vpsubb(nds, nds, src, vector_len); |
|
4545 |
evmovdqul(dst, nds, Assembler::AVX_512bit); |
|
4546 |
} else { |
|
4547 |
// worse case scenario, all regs in upper bank |
|
4548 |
evmovdqul(nds, xmm0, Assembler::AVX_512bit); |
|
4549 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4550 |
Assembler::vpsubw(xmm0, xmm0, src, vector_len); |
|
4551 |
evmovdqul(xmm0, nds, Assembler::AVX_512bit); |
|
4552 |
} |
|
4553 |
} |
|
4554 |
||
4555 |
void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { |
|
4556 |
int dst_enc = dst->encoding(); |
|
4557 |
int nds_enc = nds->encoding(); |
|
4558 |
int src_enc = src->encoding(); |
|
4559 |
if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) { |
|
4560 |
Assembler::vpsubw(dst, nds, src, vector_len); |
|
4561 |
} else if ((dst_enc < 16) && (src_enc < 16)) { |
|
4562 |
Assembler::vpsubw(dst, dst, src, vector_len); |
|
4563 |
} else if ((dst_enc < 16) && (nds_enc < 16)) { |
|
4564 |
// use nds as scratch for src |
|
4565 |
evmovdqul(nds, src, Assembler::AVX_512bit); |
|
4566 |
Assembler::vpsubw(dst, dst, nds, vector_len); |
|
4567 |
} else if ((src_enc < 16) && (nds_enc < 16)) { |
|
4568 |
// use nds as scratch for dst |
|
4569 |
evmovdqul(nds, dst, Assembler::AVX_512bit); |
|
4570 |
Assembler::vpsubw(nds, nds, src, vector_len); |
|
4571 |
evmovdqul(dst, nds, Assembler::AVX_512bit); |
|
4572 |
} else if (dst_enc < 16) { |
|
4573 |
// use nds as scatch for xmm0 to hold src |
|
4574 |
evmovdqul(nds, xmm0, Assembler::AVX_512bit); |
|
4575 |
evmovdqul(xmm0, src, Assembler::AVX_512bit); |
|
4576 |
Assembler::vpsubw(dst, dst, xmm0, vector_len); |
|
4577 |
evmovdqul(xmm0, nds, Assembler::AVX_512bit); |
|
4578 |
} else { |
|
4579 |
// worse case scenario, all regs are in the upper bank |
|
4580 |
subptr(rsp, 64); |
|
4581 |
evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit); |
|
4582 |
evmovdqul(nds, xmm0, Assembler::AVX_512bit); |
|
4583 |
evmovdqul(xmm1, src, Assembler::AVX_512bit); |
|
4584 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4585 |
Assembler::vpsubw(xmm0, xmm0, xmm1, vector_len); |
|
4586 |
evmovdqul(dst, xmm0, Assembler::AVX_512bit); |
|
4587 |
evmovdqul(xmm0, nds, Assembler::AVX_512bit); |
|
4588 |
evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit); |
|
4589 |
addptr(rsp, 64); |
|
4590 |
} |
|
4591 |
} |
|
4592 |
||
4593 |
void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { |
|
4594 |
int dst_enc = dst->encoding(); |
|
4595 |
int nds_enc = nds->encoding(); |
|
4596 |
if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) { |
|
4597 |
Assembler::vpsubw(dst, nds, src, vector_len); |
|
4598 |
} else if (dst_enc < 16) { |
|
4599 |
Assembler::vpsubw(dst, dst, src, vector_len); |
|
4600 |
} else if (nds_enc < 16) { |
|
4601 |
// implies dst_enc in upper bank with src as scratch |
|
4602 |
evmovdqul(nds, dst, Assembler::AVX_512bit); |
|
4603 |
Assembler::vpsubw(nds, nds, src, vector_len); |
|
4604 |
evmovdqul(dst, nds, Assembler::AVX_512bit); |
|
4605 |
} else { |
|
4606 |
// worse case scenario, all regs in upper bank |
|
4607 |
evmovdqul(nds, xmm0, Assembler::AVX_512bit); |
|
4608 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4609 |
Assembler::vpsubw(xmm0, xmm0, src, vector_len); |
|
4610 |
evmovdqul(xmm0, nds, Assembler::AVX_512bit); |
|
4611 |
} |
|
4612 |
} |
|
4613 |
||
4614 |
void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { |
|
4615 |
int dst_enc = dst->encoding(); |
|
4616 |
int nds_enc = nds->encoding(); |
|
4617 |
int shift_enc = shift->encoding(); |
|
4618 |
if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) { |
|
4619 |
Assembler::vpsraw(dst, nds, shift, vector_len); |
|
4620 |
} else if ((dst_enc < 16) && (shift_enc < 16)) { |
|
4621 |
Assembler::vpsraw(dst, dst, shift, vector_len); |
|
4622 |
} else if ((dst_enc < 16) && (nds_enc < 16)) { |
|
4623 |
// use nds_enc as scratch with shift |
|
4624 |
evmovdqul(nds, shift, Assembler::AVX_512bit); |
|
4625 |
Assembler::vpsraw(dst, dst, nds, vector_len); |
|
4626 |
} else if ((shift_enc < 16) && (nds_enc < 16)) { |
|
4627 |
// use nds as scratch with dst |
|
4628 |
evmovdqul(nds, dst, Assembler::AVX_512bit); |
|
4629 |
Assembler::vpsraw(nds, nds, shift, vector_len); |
|
4630 |
evmovdqul(dst, nds, Assembler::AVX_512bit); |
|
4631 |
} else if (dst_enc < 16) { |
|
4632 |
// use nds to save a copy of xmm0 and hold shift |
|
4633 |
evmovdqul(nds, xmm0, Assembler::AVX_512bit); |
|
4634 |
evmovdqul(xmm0, shift, Assembler::AVX_512bit); |
|
4635 |
Assembler::vpsraw(dst, dst, xmm0, vector_len); |
|
4636 |
evmovdqul(xmm0, nds, Assembler::AVX_512bit); |
|
4637 |
} else if (nds_enc < 16) { |
|
4638 |
// use nds as dest as temps |
|
4639 |
evmovdqul(nds, dst, Assembler::AVX_512bit); |
|
4640 |
evmovdqul(dst, xmm0, Assembler::AVX_512bit); |
|
4641 |
evmovdqul(xmm0, shift, Assembler::AVX_512bit); |
|
4642 |
Assembler::vpsraw(nds, nds, xmm0, vector_len); |
|
4643 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4644 |
evmovdqul(dst, nds, Assembler::AVX_512bit); |
|
4645 |
} else { |
|
4646 |
// worse case scenario, all regs are in the upper bank |
|
4647 |
subptr(rsp, 64); |
|
4648 |
evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit); |
|
4649 |
evmovdqul(nds, xmm0, Assembler::AVX_512bit); |
|
4650 |
evmovdqul(xmm1, shift, Assembler::AVX_512bit); |
|
4651 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4652 |
Assembler::vpsllw(xmm0, xmm0, xmm1, vector_len); |
|
4653 |
evmovdqul(xmm1, dst, Assembler::AVX_512bit); |
|
4654 |
evmovdqul(dst, xmm0, Assembler::AVX_512bit); |
|
4655 |
evmovdqul(xmm0, nds, Assembler::AVX_512bit); |
|
4656 |
evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit); |
|
4657 |
addptr(rsp, 64); |
|
4658 |
} |
|
4659 |
} |
|
4660 |
||
4661 |
void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { |
|
4662 |
int dst_enc = dst->encoding(); |
|
4663 |
int nds_enc = nds->encoding(); |
|
4664 |
if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) { |
|
4665 |
Assembler::vpsraw(dst, nds, shift, vector_len); |
|
4666 |
} else if (dst_enc < 16) { |
|
4667 |
Assembler::vpsraw(dst, dst, shift, vector_len); |
|
4668 |
} else if (nds_enc < 16) { |
|
4669 |
// use nds as scratch |
|
4670 |
evmovdqul(nds, dst, Assembler::AVX_512bit); |
|
4671 |
Assembler::vpsraw(nds, nds, shift, vector_len); |
|
4672 |
evmovdqul(dst, nds, Assembler::AVX_512bit); |
|
4673 |
} else { |
|
4674 |
// use nds as scratch for xmm0 |
|
4675 |
evmovdqul(nds, xmm0, Assembler::AVX_512bit); |
|
4676 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4677 |
Assembler::vpsraw(xmm0, xmm0, shift, vector_len); |
|
4678 |
evmovdqul(xmm0, nds, Assembler::AVX_512bit); |
|
4679 |
} |
|
4680 |
} |
|
4681 |
||
4682 |
void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { |
|
4683 |
int dst_enc = dst->encoding(); |
|
4684 |
int nds_enc = nds->encoding(); |
|
4685 |
int shift_enc = shift->encoding(); |
|
4686 |
if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) { |
|
4687 |
Assembler::vpsrlw(dst, nds, shift, vector_len); |
|
4688 |
} else if ((dst_enc < 16) && (shift_enc < 16)) { |
|
4689 |
Assembler::vpsrlw(dst, dst, shift, vector_len); |
|
4690 |
} else if ((dst_enc < 16) && (nds_enc < 16)) { |
|
4691 |
// use nds_enc as scratch with shift |
|
4692 |
evmovdqul(nds, shift, Assembler::AVX_512bit); |
|
4693 |
Assembler::vpsrlw(dst, dst, nds, vector_len); |
|
4694 |
} else if ((shift_enc < 16) && (nds_enc < 16)) { |
|
4695 |
// use nds as scratch with dst |
|
4696 |
evmovdqul(nds, dst, Assembler::AVX_512bit); |
|
4697 |
Assembler::vpsrlw(nds, nds, shift, vector_len); |
|
4698 |
evmovdqul(dst, nds, Assembler::AVX_512bit); |
|
4699 |
} else if (dst_enc < 16) { |
|
4700 |
// use nds to save a copy of xmm0 and hold shift |
|
4701 |
evmovdqul(nds, xmm0, Assembler::AVX_512bit); |
|
4702 |
evmovdqul(xmm0, shift, Assembler::AVX_512bit); |
|
4703 |
Assembler::vpsrlw(dst, dst, xmm0, vector_len); |
|
4704 |
evmovdqul(xmm0, nds, Assembler::AVX_512bit); |
|
4705 |
} else if (nds_enc < 16) { |
|
4706 |
// use nds as dest as temps |
|
4707 |
evmovdqul(nds, dst, Assembler::AVX_512bit); |
|
4708 |
evmovdqul(dst, xmm0, Assembler::AVX_512bit); |
|
4709 |
evmovdqul(xmm0, shift, Assembler::AVX_512bit); |
|
4710 |
Assembler::vpsrlw(nds, nds, xmm0, vector_len); |
|
4711 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4712 |
evmovdqul(dst, nds, Assembler::AVX_512bit); |
|
4713 |
} else { |
|
4714 |
// worse case scenario, all regs are in the upper bank |
|
4715 |
subptr(rsp, 64); |
|
4716 |
evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit); |
|
4717 |
evmovdqul(nds, xmm0, Assembler::AVX_512bit); |
|
4718 |
evmovdqul(xmm1, shift, Assembler::AVX_512bit); |
|
4719 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4720 |
Assembler::vpsllw(xmm0, xmm0, xmm1, vector_len); |
|
4721 |
evmovdqul(xmm1, dst, Assembler::AVX_512bit); |
|
4722 |
evmovdqul(dst, xmm0, Assembler::AVX_512bit); |
|
4723 |
evmovdqul(xmm0, nds, Assembler::AVX_512bit); |
|
4724 |
evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit); |
|
4725 |
addptr(rsp, 64); |
|
4726 |
} |
|
4727 |
} |
|
4728 |
||
4729 |
void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { |
|
4730 |
int dst_enc = dst->encoding(); |
|
4731 |
int nds_enc = nds->encoding(); |
|
4732 |
if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) { |
|
4733 |
Assembler::vpsrlw(dst, nds, shift, vector_len); |
|
4734 |
} else if (dst_enc < 16) { |
|
4735 |
Assembler::vpsrlw(dst, dst, shift, vector_len); |
|
4736 |
} else if (nds_enc < 16) { |
|
4737 |
// use nds as scratch |
|
4738 |
evmovdqul(nds, dst, Assembler::AVX_512bit); |
|
4739 |
Assembler::vpsrlw(nds, nds, shift, vector_len); |
|
4740 |
evmovdqul(dst, nds, Assembler::AVX_512bit); |
|
4741 |
} else { |
|
4742 |
// use nds as scratch for xmm0 |
|
4743 |
evmovdqul(nds, xmm0, Assembler::AVX_512bit); |
|
4744 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4745 |
Assembler::vpsrlw(xmm0, xmm0, shift, vector_len); |
|
4746 |
evmovdqul(xmm0, nds, Assembler::AVX_512bit); |
|
4747 |
} |
|
4748 |
} |
|
4749 |
||
4750 |
void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { |
|
4751 |
int dst_enc = dst->encoding(); |
|
4752 |
int nds_enc = nds->encoding(); |
|
4753 |
int shift_enc = shift->encoding(); |
|
4754 |
if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) { |
|
4755 |
Assembler::vpsllw(dst, nds, shift, vector_len); |
|
4756 |
} else if ((dst_enc < 16) && (shift_enc < 16)) { |
|
4757 |
Assembler::vpsllw(dst, dst, shift, vector_len); |
|
4758 |
} else if ((dst_enc < 16) && (nds_enc < 16)) { |
|
4759 |
// use nds_enc as scratch with shift |
|
4760 |
evmovdqul(nds, shift, Assembler::AVX_512bit); |
|
4761 |
Assembler::vpsllw(dst, dst, nds, vector_len); |
|
4762 |
} else if ((shift_enc < 16) && (nds_enc < 16)) { |
|
4763 |
// use nds as scratch with dst |
|
4764 |
evmovdqul(nds, dst, Assembler::AVX_512bit); |
|
4765 |
Assembler::vpsllw(nds, nds, shift, vector_len); |
|
4766 |
evmovdqul(dst, nds, Assembler::AVX_512bit); |
|
4767 |
} else if (dst_enc < 16) { |
|
4768 |
// use nds to save a copy of xmm0 and hold shift |
|
4769 |
evmovdqul(nds, xmm0, Assembler::AVX_512bit); |
|
4770 |
evmovdqul(xmm0, shift, Assembler::AVX_512bit); |
|
4771 |
Assembler::vpsllw(dst, dst, xmm0, vector_len); |
|
4772 |
evmovdqul(xmm0, nds, Assembler::AVX_512bit); |
|
4773 |
} else if (nds_enc < 16) { |
|
4774 |
// use nds as dest as temps |
|
4775 |
evmovdqul(nds, dst, Assembler::AVX_512bit); |
|
4776 |
evmovdqul(dst, xmm0, Assembler::AVX_512bit); |
|
4777 |
evmovdqul(xmm0, shift, Assembler::AVX_512bit); |
|
4778 |
Assembler::vpsllw(nds, nds, xmm0, vector_len); |
|
4779 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4780 |
evmovdqul(dst, nds, Assembler::AVX_512bit); |
|
4781 |
} else { |
|
4782 |
// worse case scenario, all regs are in the upper bank |
|
4783 |
subptr(rsp, 64); |
|
4784 |
evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit); |
|
4785 |
evmovdqul(nds, xmm0, Assembler::AVX_512bit); |
|
4786 |
evmovdqul(xmm1, shift, Assembler::AVX_512bit); |
|
4787 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4788 |
Assembler::vpsllw(xmm0, xmm0, xmm1, vector_len); |
|
4789 |
evmovdqul(xmm1, dst, Assembler::AVX_512bit); |
|
4790 |
evmovdqul(dst, xmm0, Assembler::AVX_512bit); |
|
4791 |
evmovdqul(xmm0, nds, Assembler::AVX_512bit); |
|
4792 |
evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit); |
|
4793 |
addptr(rsp, 64); |
|
4794 |
} |
|
4795 |
} |
|
4796 |
||
4797 |
void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { |
|
4798 |
int dst_enc = dst->encoding(); |
|
4799 |
int nds_enc = nds->encoding(); |
|
4800 |
if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) { |
|
4801 |
Assembler::vpsllw(dst, nds, shift, vector_len); |
|
4802 |
} else if (dst_enc < 16) { |
|
4803 |
Assembler::vpsllw(dst, dst, shift, vector_len); |
|
4804 |
} else if (nds_enc < 16) { |
|
4805 |
// use nds as scratch |
|
4806 |
evmovdqul(nds, dst, Assembler::AVX_512bit); |
|
4807 |
Assembler::vpsllw(nds, nds, shift, vector_len); |
|
4808 |
evmovdqul(dst, nds, Assembler::AVX_512bit); |
|
4809 |
} else { |
|
4810 |
// use nds as scratch for xmm0 |
|
4811 |
evmovdqul(nds, xmm0, Assembler::AVX_512bit); |
|
4812 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4813 |
Assembler::vpsllw(xmm0, xmm0, shift, vector_len); |
|
4814 |
evmovdqul(xmm0, nds, Assembler::AVX_512bit); |
|
4815 |
} |
|
4816 |
} |
|
4817 |
||
34203 | 4818 |
void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) { |
4819 |
int dst_enc = dst->encoding(); |
|
4820 |
int src_enc = src->encoding(); |
|
4821 |
if ((dst_enc < 16) && (src_enc < 16)) { |
|
4822 |
Assembler::vptest(dst, src); |
|
4823 |
} else if (src_enc < 16) { |
|
4824 |
subptr(rsp, 64); |
|
4825 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
4826 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4827 |
Assembler::vptest(xmm0, src); |
|
4828 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
4829 |
addptr(rsp, 64); |
|
4830 |
} else if (dst_enc < 16) { |
|
4831 |
subptr(rsp, 64); |
|
4832 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
4833 |
evmovdqul(xmm0, src, Assembler::AVX_512bit); |
|
4834 |
Assembler::vptest(dst, xmm0); |
|
4835 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
4836 |
addptr(rsp, 64); |
|
4837 |
} else { |
|
4838 |
subptr(rsp, 64); |
|
4839 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
4840 |
subptr(rsp, 64); |
|
4841 |
evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit); |
|
4842 |
movdqu(xmm0, src); |
|
4843 |
movdqu(xmm1, dst); |
|
4844 |
Assembler::vptest(xmm1, xmm0); |
|
4845 |
evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit); |
|
4846 |
addptr(rsp, 64); |
|
4847 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
4848 |
addptr(rsp, 64); |
|
4849 |
} |
|
4850 |
} |
|
4851 |
||
34162 | 4852 |
// This instruction exists within macros, ergo we cannot control its input |
4853 |
// when emitted through those patterns. |
|
4854 |
void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) { |
|
4855 |
if (VM_Version::supports_avx512nobw()) { |
|
4856 |
int dst_enc = dst->encoding(); |
|
4857 |
int src_enc = src->encoding(); |
|
4858 |
if (dst_enc == src_enc) { |
|
4859 |
if (dst_enc < 16) { |
|
4860 |
Assembler::punpcklbw(dst, src); |
|
4861 |
} else { |
|
4862 |
subptr(rsp, 64); |
|
4863 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
4864 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4865 |
Assembler::punpcklbw(xmm0, xmm0); |
|
4866 |
evmovdqul(dst, xmm0, Assembler::AVX_512bit); |
|
4867 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
4868 |
addptr(rsp, 64); |
|
4869 |
} |
|
4870 |
} else { |
|
4871 |
if ((src_enc < 16) && (dst_enc < 16)) { |
|
4872 |
Assembler::punpcklbw(dst, src); |
|
4873 |
} else if (src_enc < 16) { |
|
4874 |
subptr(rsp, 64); |
|
4875 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
4876 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4877 |
Assembler::punpcklbw(xmm0, src); |
|
4878 |
evmovdqul(dst, xmm0, Assembler::AVX_512bit); |
|
4879 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
4880 |
addptr(rsp, 64); |
|
4881 |
} else if (dst_enc < 16) { |
|
4882 |
subptr(rsp, 64); |
|
4883 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
4884 |
evmovdqul(xmm0, src, Assembler::AVX_512bit); |
|
4885 |
Assembler::punpcklbw(dst, xmm0); |
|
4886 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
4887 |
addptr(rsp, 64); |
|
4888 |
} else { |
|
4889 |
subptr(rsp, 64); |
|
4890 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
4891 |
subptr(rsp, 64); |
|
4892 |
evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit); |
|
4893 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4894 |
evmovdqul(xmm1, src, Assembler::AVX_512bit); |
|
4895 |
Assembler::punpcklbw(xmm0, xmm1); |
|
4896 |
evmovdqul(dst, xmm0, Assembler::AVX_512bit); |
|
4897 |
evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit); |
|
4898 |
addptr(rsp, 64); |
|
4899 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
4900 |
addptr(rsp, 64); |
|
4901 |
} |
|
4902 |
} |
|
4903 |
} else { |
|
4904 |
Assembler::punpcklbw(dst, src); |
|
4905 |
} |
|
4906 |
} |
|
4907 |
||
4908 |
// This instruction exists within macros, ergo we cannot control its input |
|
4909 |
// when emitted through those patterns. |
|
4910 |
void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) { |
|
4911 |
if (VM_Version::supports_avx512nobw()) { |
|
4912 |
int dst_enc = dst->encoding(); |
|
4913 |
int src_enc = src->encoding(); |
|
4914 |
if (dst_enc == src_enc) { |
|
4915 |
if (dst_enc < 16) { |
|
4916 |
Assembler::pshuflw(dst, src, mode); |
|
4917 |
} else { |
|
4918 |
subptr(rsp, 64); |
|
4919 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
4920 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4921 |
Assembler::pshuflw(xmm0, xmm0, mode); |
|
4922 |
evmovdqul(dst, xmm0, Assembler::AVX_512bit); |
|
4923 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
4924 |
addptr(rsp, 64); |
|
4925 |
} |
|
4926 |
} else { |
|
4927 |
if ((src_enc < 16) && (dst_enc < 16)) { |
|
4928 |
Assembler::pshuflw(dst, src, mode); |
|
4929 |
} else if (src_enc < 16) { |
|
4930 |
subptr(rsp, 64); |
|
4931 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
4932 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4933 |
Assembler::pshuflw(xmm0, src, mode); |
|
4934 |
evmovdqul(dst, xmm0, Assembler::AVX_512bit); |
|
4935 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
4936 |
addptr(rsp, 64); |
|
4937 |
} else if (dst_enc < 16) { |
|
4938 |
subptr(rsp, 64); |
|
4939 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
4940 |
evmovdqul(xmm0, src, Assembler::AVX_512bit); |
|
4941 |
Assembler::pshuflw(dst, xmm0, mode); |
|
4942 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
4943 |
addptr(rsp, 64); |
|
4944 |
} else { |
|
4945 |
subptr(rsp, 64); |
|
4946 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
|
4947 |
subptr(rsp, 64); |
|
4948 |
evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit); |
|
4949 |
evmovdqul(xmm0, dst, Assembler::AVX_512bit); |
|
4950 |
evmovdqul(xmm1, src, Assembler::AVX_512bit); |
|
4951 |
Assembler::pshuflw(xmm0, xmm1, mode); |
|
4952 |
evmovdqul(dst, xmm0, Assembler::AVX_512bit); |
|
4953 |
evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit); |
|
4954 |
addptr(rsp, 64); |
|
4955 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
|
4956 |
addptr(rsp, 64); |
|
4957 |
} |
|
4958 |
} |
|
4959 |
} else { |
|
4960 |
Assembler::pshuflw(dst, src, mode); |
|
4961 |
} |
|
4962 |
} |
|
4963 |
||
30624 | 4964 |
void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) { |
14626 | 4965 |
if (reachable(src)) { |
30624 | 4966 |
vandpd(dst, nds, as_Address(src), vector_len); |
14626 | 4967 |
} else { |
4968 |
lea(rscratch1, src); |
|
30624 | 4969 |
vandpd(dst, nds, Address(rscratch1, 0), vector_len); |
4970 |
} |
|
4971 |
} |
|
4972 |
||
4973 |
void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) { |
|
14626 | 4974 |
if (reachable(src)) { |
30624 | 4975 |
vandps(dst, nds, as_Address(src), vector_len); |
14626 | 4976 |
} else { |
4977 |
lea(rscratch1, src); |
|
30624 | 4978 |
vandps(dst, nds, Address(rscratch1, 0), vector_len); |
14626 | 4979 |
} |
4980 |
} |
|
4981 |
||
4982 |
void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) { |
|
4983 |
if (reachable(src)) { |
|
4984 |
vdivsd(dst, nds, as_Address(src)); |
|
4985 |
} else { |
|
4986 |
lea(rscratch1, src); |
|
4987 |
vdivsd(dst, nds, Address(rscratch1, 0)); |
|
4988 |
} |
|
4989 |
} |
|
4990 |
||
4991 |
void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src) { |
|
4992 |
if (reachable(src)) { |
|
4993 |
vdivss(dst, nds, as_Address(src)); |
|
4994 |
} else { |
|
4995 |
lea(rscratch1, src); |
|
4996 |
vdivss(dst, nds, Address(rscratch1, 0)); |
|
4997 |
} |
|
4998 |
} |
|
4999 |
||
5000 |
void MacroAssembler::vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) { |
|
5001 |
if (reachable(src)) { |
|
5002 |
vmulsd(dst, nds, as_Address(src)); |
|
5003 |
} else { |
|
5004 |
lea(rscratch1, src); |
|
5005 |
vmulsd(dst, nds, Address(rscratch1, 0)); |
|
5006 |
} |
|
5007 |
} |
|
5008 |
||
5009 |
void MacroAssembler::vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src) { |
|
5010 |
if (reachable(src)) { |
|
5011 |
vmulss(dst, nds, as_Address(src)); |
|
5012 |
} else { |
|
5013 |
lea(rscratch1, src); |
|
5014 |
vmulss(dst, nds, Address(rscratch1, 0)); |
|
5015 |
} |
|
5016 |
} |
|
5017 |
||
5018 |
void MacroAssembler::vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) { |
|
5019 |
if (reachable(src)) { |
|
5020 |
vsubsd(dst, nds, as_Address(src)); |
|
5021 |
} else { |
|
5022 |
lea(rscratch1, src); |
|
5023 |
vsubsd(dst, nds, Address(rscratch1, 0)); |
|
5024 |
} |
|
5025 |
} |
|
5026 |
||
5027 |
void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src) { |
|
5028 |
if (reachable(src)) { |
|
5029 |
vsubss(dst, nds, as_Address(src)); |
|
5030 |
} else { |
|
5031 |
lea(rscratch1, src); |
|
5032 |
vsubss(dst, nds, Address(rscratch1, 0)); |
|
5033 |
} |
|
5034 |
} |
|
5035 |
||
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5036 |
void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src) { |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5037 |
int nds_enc = nds->encoding(); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5038 |
int dst_enc = dst->encoding(); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5039 |
bool dst_upper_bank = (dst_enc > 15); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5040 |
bool nds_upper_bank = (nds_enc > 15); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5041 |
if (VM_Version::supports_avx512novl() && |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5042 |
(nds_upper_bank || dst_upper_bank)) { |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5043 |
if (dst_upper_bank) { |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5044 |
subptr(rsp, 64); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5045 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5046 |
movflt(xmm0, nds); |
34162 | 5047 |
vxorps(xmm0, xmm0, src, Assembler::AVX_128bit); |
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5048 |
movflt(dst, xmm0); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5049 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5050 |
addptr(rsp, 64); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5051 |
} else { |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5052 |
movflt(dst, nds); |
34162 | 5053 |
vxorps(dst, dst, src, Assembler::AVX_128bit); |
5054 |
} |
|
5055 |
} else { |
|
5056 |
vxorps(dst, nds, src, Assembler::AVX_128bit); |
|
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5057 |
} |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5058 |
} |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5059 |
|
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5060 |
void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src) { |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5061 |
int nds_enc = nds->encoding(); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5062 |
int dst_enc = dst->encoding(); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5063 |
bool dst_upper_bank = (dst_enc > 15); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5064 |
bool nds_upper_bank = (nds_enc > 15); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5065 |
if (VM_Version::supports_avx512novl() && |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5066 |
(nds_upper_bank || dst_upper_bank)) { |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5067 |
if (dst_upper_bank) { |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5068 |
subptr(rsp, 64); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5069 |
evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5070 |
movdbl(xmm0, nds); |
34162 | 5071 |
vxorpd(xmm0, xmm0, src, Assembler::AVX_128bit); |
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5072 |
movdbl(dst, xmm0); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5073 |
evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5074 |
addptr(rsp, 64); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5075 |
} else { |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5076 |
movdbl(dst, nds); |
34162 | 5077 |
vxorpd(dst, dst, src, Assembler::AVX_128bit); |
5078 |
} |
|
5079 |
} else { |
|
5080 |
vxorpd(dst, nds, src, Assembler::AVX_128bit); |
|
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5081 |
} |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5082 |
} |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5083 |
|
30624 | 5084 |
void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) { |
14626 | 5085 |
if (reachable(src)) { |
30624 | 5086 |
vxorpd(dst, nds, as_Address(src), vector_len); |
14626 | 5087 |
} else { |
5088 |
lea(rscratch1, src); |
|
30624 | 5089 |
vxorpd(dst, nds, Address(rscratch1, 0), vector_len); |
5090 |
} |
|
5091 |
} |
|
5092 |
||
5093 |
void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) { |
|
14626 | 5094 |
if (reachable(src)) { |
30624 | 5095 |
vxorps(dst, nds, as_Address(src), vector_len); |
14626 | 5096 |
} else { |
5097 |
lea(rscratch1, src); |
|
30624 | 5098 |
vxorps(dst, nds, Address(rscratch1, 0), vector_len); |
14626 | 5099 |
} |
5100 |
} |
|
5101 |
||
5102 |
||
5103 |
////////////////////////////////////////////////////////////////////////////////// |
|
15482
470d0b0c09f1
8005915: Unify SERIALGC and INCLUDE_ALTERNATE_GCS
jprovino
parents:
15117
diff
changeset
|
5104 |
#if INCLUDE_ALL_GCS |
14626 | 5105 |
|
5106 |
void MacroAssembler::g1_write_barrier_pre(Register obj, |
|
5107 |
Register pre_val, |
|
5108 |
Register thread, |
|
5109 |
Register tmp, |
|
5110 |
bool tosca_live, |
|
5111 |
bool expand_call) { |
|
5112 |
||
5113 |
// If expand_call is true then we expand the call_VM_leaf macro |
|
5114 |
// directly to skip generating the check by |
|
5115 |
// InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp. |
|
5116 |
||
5117 |
#ifdef _LP64 |
|
5118 |
assert(thread == r15_thread, "must be"); |
|
5119 |
#endif // _LP64 |
|
5120 |
||
5121 |
Label done; |
|
5122 |
Label runtime; |
|
5123 |
||
5124 |
assert(pre_val != noreg, "check this code"); |
|
5125 |
||
5126 |
if (obj != noreg) { |
|
5127 |
assert_different_registers(obj, pre_val, tmp); |
|
5128 |
assert(pre_val != rax, "check this code"); |
|
5129 |
} |
|
5130 |
||
5131 |
Address in_progress(thread, in_bytes(JavaThread::satb_mark_queue_offset() + |
|
34148
6efbc7ffd767
8143014: Access PtrQueue member offsets through derived classes
kbarrett
parents:
33639
diff
changeset
|
5132 |
SATBMarkQueue::byte_offset_of_active())); |
14626 | 5133 |
Address index(thread, in_bytes(JavaThread::satb_mark_queue_offset() + |
34148
6efbc7ffd767
8143014: Access PtrQueue member offsets through derived classes
kbarrett
parents:
33639
diff
changeset
|
5134 |
SATBMarkQueue::byte_offset_of_index())); |
14626 | 5135 |
Address buffer(thread, in_bytes(JavaThread::satb_mark_queue_offset() + |
34148
6efbc7ffd767
8143014: Access PtrQueue member offsets through derived classes
kbarrett
parents:
33639
diff
changeset
|
5136 |
SATBMarkQueue::byte_offset_of_buf())); |
14626 | 5137 |
|
5138 |
||
5139 |
// Is marking active? |
|
34148
6efbc7ffd767
8143014: Access PtrQueue member offsets through derived classes
kbarrett
parents:
33639
diff
changeset
|
5140 |
if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { |
14626 | 5141 |
cmpl(in_progress, 0); |
5142 |
} else { |
|
34148
6efbc7ffd767
8143014: Access PtrQueue member offsets through derived classes
kbarrett
parents:
33639
diff
changeset
|
5143 |
assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); |
14626 | 5144 |
cmpb(in_progress, 0); |
5145 |
} |
|
5146 |
jcc(Assembler::equal, done); |
|
5147 |
||
5148 |
// Do we need to load the previous value? |
|
5149 |
if (obj != noreg) { |
|
5150 |
load_heap_oop(pre_val, Address(obj, 0)); |
|
5151 |
} |
|
5152 |
||
5153 |
// Is the previous value null? |
|
5154 |
cmpptr(pre_val, (int32_t) NULL_WORD); |
|
5155 |
jcc(Assembler::equal, done); |
|
5156 |
||
5157 |
// Can we store original value in the thread's buffer? |
|
5158 |
// Is index == 0? |
|
5159 |
// (The index field is typed as size_t.) |
|
5160 |
||
5161 |
movptr(tmp, index); // tmp := *index_adr |
|
5162 |
cmpptr(tmp, 0); // tmp == 0? |
|
5163 |
jcc(Assembler::equal, runtime); // If yes, goto runtime |
|
5164 |
||
5165 |
subptr(tmp, wordSize); // tmp := tmp - wordSize |
|
5166 |
movptr(index, tmp); // *index_adr := tmp |
|
5167 |
addptr(tmp, buffer); // tmp := tmp + *buffer_adr |
|
5168 |
||
5169 |
// Record the previous value |
|
5170 |
movptr(Address(tmp, 0), pre_val); |
|
5171 |
jmp(done); |
|
5172 |
||
5173 |
bind(runtime); |
|
5174 |
// save the live input values |
|
5175 |
if(tosca_live) push(rax); |
|
5176 |
||
5177 |
if (obj != noreg && obj != rax) |
|
5178 |
push(obj); |
|
5179 |
||
5180 |
if (pre_val != rax) |
|
5181 |
push(pre_val); |
|
5182 |
||
5183 |
// Calling the runtime using the regular call_VM_leaf mechanism generates |
|
5184 |
// code (generated by InterpreterMacroAssember::call_VM_leaf_base) |
|
5185 |
// that checks that the *(ebp+frame::interpreter_frame_last_sp) == NULL. |
|
5186 |
// |
|
5187 |
// If we care generating the pre-barrier without a frame (e.g. in the |
|
5188 |
// intrinsified Reference.get() routine) then ebp might be pointing to |
|
5189 |
// the caller frame and so this check will most likely fail at runtime. |
|
5190 |
// |
|
5191 |
// Expanding the call directly bypasses the generation of the check. |
|
5192 |
// So when we do not have have a full interpreter frame on the stack |
|
5193 |
// expand_call should be passed true. |
|
5194 |
||
5195 |
NOT_LP64( push(thread); ) |
|
5196 |
||
5197 |
if (expand_call) { |
|
5198 |
LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); ) |
|
5199 |
pass_arg1(this, thread); |
|
5200 |
pass_arg0(this, pre_val); |
|
5201 |
MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), 2); |
|
5202 |
} else { |
|
5203 |
call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), pre_val, thread); |
|
5204 |
} |
|
5205 |
||
5206 |
NOT_LP64( pop(thread); ) |
|
5207 |
||
5208 |
// save the live input values |
|
5209 |
if (pre_val != rax) |
|
5210 |
pop(pre_val); |
|
5211 |
||
5212 |
if (obj != noreg && obj != rax) |
|
5213 |
pop(obj); |
|
5214 |
||
5215 |
if(tosca_live) pop(rax); |
|
5216 |
||
5217 |
bind(done); |
|
5218 |
} |
|
5219 |
||
5220 |
void MacroAssembler::g1_write_barrier_post(Register store_addr, |
|
5221 |
Register new_val, |
|
5222 |
Register thread, |
|
5223 |
Register tmp, |
|
5224 |
Register tmp2) { |
|
5225 |
#ifdef _LP64 |
|
5226 |
assert(thread == r15_thread, "must be"); |
|
5227 |
#endif // _LP64 |
|
5228 |
||
5229 |
Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() + |
|
34148
6efbc7ffd767
8143014: Access PtrQueue member offsets through derived classes
kbarrett
parents:
33639
diff
changeset
|
5230 |
DirtyCardQueue::byte_offset_of_index())); |
14626 | 5231 |
Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() + |
34148
6efbc7ffd767
8143014: Access PtrQueue member offsets through derived classes
kbarrett
parents:
33639
diff
changeset
|
5232 |
DirtyCardQueue::byte_offset_of_buf())); |
14626 | 5233 |
|
29325 | 5234 |
CardTableModRefBS* ct = |
5235 |
barrier_set_cast<CardTableModRefBS>(Universe::heap()->barrier_set()); |
|
21923
5cd9eb764fe9
8028109: compiler/codecache/CheckReservedInitialCodeCacheSizeArgOrder.java crashes in RT_Baseline
anoll
parents:
21528
diff
changeset
|
5236 |
assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code"); |
5cd9eb764fe9
8028109: compiler/codecache/CheckReservedInitialCodeCacheSizeArgOrder.java crashes in RT_Baseline
anoll
parents:
21528
diff
changeset
|
5237 |
|
14626 | 5238 |
Label done; |
5239 |
Label runtime; |
|
5240 |
||
5241 |
// Does store cross heap regions? |
|
5242 |
||
5243 |
movptr(tmp, store_addr); |
|
5244 |
xorptr(tmp, new_val); |
|
5245 |
shrptr(tmp, HeapRegion::LogOfHRGrainBytes); |
|
5246 |
jcc(Assembler::equal, done); |
|
5247 |
||
5248 |
// crosses regions, storing NULL? |
|
5249 |
||
5250 |
cmpptr(new_val, (int32_t) NULL_WORD); |
|
5251 |
jcc(Assembler::equal, done); |
|
5252 |
||
5253 |
// storing region crossing non-NULL, is card already dirty? |
|
5254 |
||
5255 |
const Register card_addr = tmp; |
|
21923
5cd9eb764fe9
8028109: compiler/codecache/CheckReservedInitialCodeCacheSizeArgOrder.java crashes in RT_Baseline
anoll
parents:
21528
diff
changeset
|
5256 |
const Register cardtable = tmp2; |
5cd9eb764fe9
8028109: compiler/codecache/CheckReservedInitialCodeCacheSizeArgOrder.java crashes in RT_Baseline
anoll
parents:
21528
diff
changeset
|
5257 |
|
5cd9eb764fe9
8028109: compiler/codecache/CheckReservedInitialCodeCacheSizeArgOrder.java crashes in RT_Baseline
anoll
parents:
21528
diff
changeset
|
5258 |
movptr(card_addr, store_addr); |
5cd9eb764fe9
8028109: compiler/codecache/CheckReservedInitialCodeCacheSizeArgOrder.java crashes in RT_Baseline
anoll
parents:
21528
diff
changeset
|
5259 |
shrptr(card_addr, CardTableModRefBS::card_shift); |
5cd9eb764fe9
8028109: compiler/codecache/CheckReservedInitialCodeCacheSizeArgOrder.java crashes in RT_Baseline
anoll
parents:
21528
diff
changeset
|
5260 |
// Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT |
5cd9eb764fe9
8028109: compiler/codecache/CheckReservedInitialCodeCacheSizeArgOrder.java crashes in RT_Baseline
anoll
parents:
21528
diff
changeset
|
5261 |
// a valid address and therefore is not properly handled by the relocation code. |
5cd9eb764fe9
8028109: compiler/codecache/CheckReservedInitialCodeCacheSizeArgOrder.java crashes in RT_Baseline
anoll
parents:
21528
diff
changeset
|
5262 |
movptr(cardtable, (intptr_t)ct->byte_map_base); |
5cd9eb764fe9
8028109: compiler/codecache/CheckReservedInitialCodeCacheSizeArgOrder.java crashes in RT_Baseline
anoll
parents:
21528
diff
changeset
|
5263 |
addptr(card_addr, cardtable); |
5cd9eb764fe9
8028109: compiler/codecache/CheckReservedInitialCodeCacheSizeArgOrder.java crashes in RT_Baseline
anoll
parents:
21528
diff
changeset
|
5264 |
|
20403
45a89fbcd8f7
8014555: G1: Memory ordering problem with Conc refinement and card marking
mgerdin
parents:
19979
diff
changeset
|
5265 |
cmpb(Address(card_addr, 0), (int)G1SATBCardTableModRefBS::g1_young_card_val()); |
14626 | 5266 |
jcc(Assembler::equal, done); |
5267 |
||
20403
45a89fbcd8f7
8014555: G1: Memory ordering problem with Conc refinement and card marking
mgerdin
parents:
19979
diff
changeset
|
5268 |
membar(Assembler::Membar_mask_bits(Assembler::StoreLoad)); |
45a89fbcd8f7
8014555: G1: Memory ordering problem with Conc refinement and card marking
mgerdin
parents:
19979
diff
changeset
|
5269 |
cmpb(Address(card_addr, 0), (int)CardTableModRefBS::dirty_card_val()); |
45a89fbcd8f7
8014555: G1: Memory ordering problem with Conc refinement and card marking
mgerdin
parents:
19979
diff
changeset
|
5270 |
jcc(Assembler::equal, done); |
45a89fbcd8f7
8014555: G1: Memory ordering problem with Conc refinement and card marking
mgerdin
parents:
19979
diff
changeset
|
5271 |
|
45a89fbcd8f7
8014555: G1: Memory ordering problem with Conc refinement and card marking
mgerdin
parents:
19979
diff
changeset
|
5272 |
|
14626 | 5273 |
// storing a region crossing, non-NULL oop, card is clean. |
5274 |
// dirty card and log. |
|
5275 |
||
20403
45a89fbcd8f7
8014555: G1: Memory ordering problem with Conc refinement and card marking
mgerdin
parents:
19979
diff
changeset
|
5276 |
movb(Address(card_addr, 0), (int)CardTableModRefBS::dirty_card_val()); |
14626 | 5277 |
|
5278 |
cmpl(queue_index, 0); |
|
5279 |
jcc(Assembler::equal, runtime); |
|
5280 |
subl(queue_index, wordSize); |
|
5281 |
movptr(tmp2, buffer); |
|
5282 |
#ifdef _LP64 |
|
5283 |
movslq(rscratch1, queue_index); |
|
5284 |
addq(tmp2, rscratch1); |
|
5285 |
movq(Address(tmp2, 0), card_addr); |
|
5286 |
#else |
|
5287 |
addl(tmp2, queue_index); |
|
21923
5cd9eb764fe9
8028109: compiler/codecache/CheckReservedInitialCodeCacheSizeArgOrder.java crashes in RT_Baseline
anoll
parents:
21528
diff
changeset
|
5288 |
movl(Address(tmp2, 0), card_addr); |
14626 | 5289 |
#endif |
5290 |
jmp(done); |
|
5291 |
||
5292 |
bind(runtime); |
|
5293 |
// save the live input values |
|
5294 |
push(store_addr); |
|
5295 |
push(new_val); |
|
5296 |
#ifdef _LP64 |
|
5297 |
call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, r15_thread); |
|
5298 |
#else |
|
5299 |
push(thread); |
|
5300 |
call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, thread); |
|
5301 |
pop(thread); |
|
5302 |
#endif |
|
5303 |
pop(new_val); |
|
5304 |
pop(store_addr); |
|
5305 |
||
5306 |
bind(done); |
|
5307 |
} |
|
5308 |
||
15482
470d0b0c09f1
8005915: Unify SERIALGC and INCLUDE_ALTERNATE_GCS
jprovino
parents:
15117
diff
changeset
|
5309 |
#endif // INCLUDE_ALL_GCS |
14626 | 5310 |
////////////////////////////////////////////////////////////////////////////////// |
5311 |
||
5312 |
||
31368
2cb1abbda511
8078438: Interpreter should support conditional card marks (UseCondCardMark) on x86 and aarch64
shade
parents:
31129
diff
changeset
|
5313 |
void MacroAssembler::store_check(Register obj, Address dst) { |
2cb1abbda511
8078438: Interpreter should support conditional card marks (UseCondCardMark) on x86 and aarch64
shade
parents:
31129
diff
changeset
|
5314 |
store_check(obj); |
2cb1abbda511
8078438: Interpreter should support conditional card marks (UseCondCardMark) on x86 and aarch64
shade
parents:
31129
diff
changeset
|
5315 |
} |
2cb1abbda511
8078438: Interpreter should support conditional card marks (UseCondCardMark) on x86 and aarch64
shade
parents:
31129
diff
changeset
|
5316 |
|
14626 | 5317 |
void MacroAssembler::store_check(Register obj) { |
5318 |
// Does a store check for the oop in register obj. The content of |
|
5319 |
// register obj is destroyed afterwards. |
|
5320 |
BarrierSet* bs = Universe::heap()->barrier_set(); |
|
32596
8feecdee3156
8072817: CardTableExtension kind() should be BarrierSet::CardTableExtension
kbarrett
parents:
32203
diff
changeset
|
5321 |
assert(bs->kind() == BarrierSet::CardTableForRS || |
8feecdee3156
8072817: CardTableExtension kind() should be BarrierSet::CardTableExtension
kbarrett
parents:
32203
diff
changeset
|
5322 |
bs->kind() == BarrierSet::CardTableExtension, |
8feecdee3156
8072817: CardTableExtension kind() should be BarrierSet::CardTableExtension
kbarrett
parents:
32203
diff
changeset
|
5323 |
"Wrong barrier set kind"); |
31368
2cb1abbda511
8078438: Interpreter should support conditional card marks (UseCondCardMark) on x86 and aarch64
shade
parents:
31129
diff
changeset
|
5324 |
|
29325 | 5325 |
CardTableModRefBS* ct = barrier_set_cast<CardTableModRefBS>(bs); |
14626 | 5326 |
assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code"); |
5327 |
||
31368
2cb1abbda511
8078438: Interpreter should support conditional card marks (UseCondCardMark) on x86 and aarch64
shade
parents:
31129
diff
changeset
|
5328 |
shrptr(obj, CardTableModRefBS::card_shift); |
2cb1abbda511
8078438: Interpreter should support conditional card marks (UseCondCardMark) on x86 and aarch64
shade
parents:
31129
diff
changeset
|
5329 |
|
2cb1abbda511
8078438: Interpreter should support conditional card marks (UseCondCardMark) on x86 and aarch64
shade
parents:
31129
diff
changeset
|
5330 |
Address card_addr; |
2cb1abbda511
8078438: Interpreter should support conditional card marks (UseCondCardMark) on x86 and aarch64
shade
parents:
31129
diff
changeset
|
5331 |
|
14626 | 5332 |
// The calculation for byte_map_base is as follows: |
5333 |
// byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift); |
|
21923
5cd9eb764fe9
8028109: compiler/codecache/CheckReservedInitialCodeCacheSizeArgOrder.java crashes in RT_Baseline
anoll
parents:
21528
diff
changeset
|
5334 |
// So this essentially converts an address to a displacement and it will |
5cd9eb764fe9
8028109: compiler/codecache/CheckReservedInitialCodeCacheSizeArgOrder.java crashes in RT_Baseline
anoll
parents:
21528
diff
changeset
|
5335 |
// never need to be relocated. On 64bit however the value may be too |
5cd9eb764fe9
8028109: compiler/codecache/CheckReservedInitialCodeCacheSizeArgOrder.java crashes in RT_Baseline
anoll
parents:
21528
diff
changeset
|
5336 |
// large for a 32bit displacement. |
14626 | 5337 |
intptr_t disp = (intptr_t) ct->byte_map_base; |
5338 |
if (is_simm32(disp)) { |
|
31368
2cb1abbda511
8078438: Interpreter should support conditional card marks (UseCondCardMark) on x86 and aarch64
shade
parents:
31129
diff
changeset
|
5339 |
card_addr = Address(noreg, obj, Address::times_1, disp); |
14626 | 5340 |
} else { |
21923
5cd9eb764fe9
8028109: compiler/codecache/CheckReservedInitialCodeCacheSizeArgOrder.java crashes in RT_Baseline
anoll
parents:
21528
diff
changeset
|
5341 |
// By doing it as an ExternalAddress 'disp' could be converted to a rip-relative |
5cd9eb764fe9
8028109: compiler/codecache/CheckReservedInitialCodeCacheSizeArgOrder.java crashes in RT_Baseline
anoll
parents:
21528
diff
changeset
|
5342 |
// displacement and done in a single instruction given favorable mapping and a |
5cd9eb764fe9
8028109: compiler/codecache/CheckReservedInitialCodeCacheSizeArgOrder.java crashes in RT_Baseline
anoll
parents:
21528
diff
changeset
|
5343 |
// smarter version of as_Address. However, 'ExternalAddress' generates a relocation |
5cd9eb764fe9
8028109: compiler/codecache/CheckReservedInitialCodeCacheSizeArgOrder.java crashes in RT_Baseline
anoll
parents:
21528
diff
changeset
|
5344 |
// entry and that entry is not properly handled by the relocation code. |
5cd9eb764fe9
8028109: compiler/codecache/CheckReservedInitialCodeCacheSizeArgOrder.java crashes in RT_Baseline
anoll
parents:
21528
diff
changeset
|
5345 |
AddressLiteral cardtable((address)ct->byte_map_base, relocInfo::none); |
14626 | 5346 |
Address index(noreg, obj, Address::times_1); |
31368
2cb1abbda511
8078438: Interpreter should support conditional card marks (UseCondCardMark) on x86 and aarch64
shade
parents:
31129
diff
changeset
|
5347 |
card_addr = as_Address(ArrayAddress(cardtable, index)); |
2cb1abbda511
8078438: Interpreter should support conditional card marks (UseCondCardMark) on x86 and aarch64
shade
parents:
31129
diff
changeset
|
5348 |
} |
2cb1abbda511
8078438: Interpreter should support conditional card marks (UseCondCardMark) on x86 and aarch64
shade
parents:
31129
diff
changeset
|
5349 |
|
2cb1abbda511
8078438: Interpreter should support conditional card marks (UseCondCardMark) on x86 and aarch64
shade
parents:
31129
diff
changeset
|
5350 |
int dirty = CardTableModRefBS::dirty_card_val(); |
2cb1abbda511
8078438: Interpreter should support conditional card marks (UseCondCardMark) on x86 and aarch64
shade
parents:
31129
diff
changeset
|
5351 |
if (UseCondCardMark) { |
2cb1abbda511
8078438: Interpreter should support conditional card marks (UseCondCardMark) on x86 and aarch64
shade
parents:
31129
diff
changeset
|
5352 |
Label L_already_dirty; |
31369
0c3dcc865a1c
8079315: UseCondCardMark broken in conjunction with CMS precleaning on x86
aph
parents:
31368
diff
changeset
|
5353 |
if (UseConcMarkSweepGC) { |
0c3dcc865a1c
8079315: UseCondCardMark broken in conjunction with CMS precleaning on x86
aph
parents:
31368
diff
changeset
|
5354 |
membar(Assembler::StoreLoad); |
0c3dcc865a1c
8079315: UseCondCardMark broken in conjunction with CMS precleaning on x86
aph
parents:
31368
diff
changeset
|
5355 |
} |
31368
2cb1abbda511
8078438: Interpreter should support conditional card marks (UseCondCardMark) on x86 and aarch64
shade
parents:
31129
diff
changeset
|
5356 |
cmpb(card_addr, dirty); |
2cb1abbda511
8078438: Interpreter should support conditional card marks (UseCondCardMark) on x86 and aarch64
shade
parents:
31129
diff
changeset
|
5357 |
jcc(Assembler::equal, L_already_dirty); |
2cb1abbda511
8078438: Interpreter should support conditional card marks (UseCondCardMark) on x86 and aarch64
shade
parents:
31129
diff
changeset
|
5358 |
movb(card_addr, dirty); |
2cb1abbda511
8078438: Interpreter should support conditional card marks (UseCondCardMark) on x86 and aarch64
shade
parents:
31129
diff
changeset
|
5359 |
bind(L_already_dirty); |
2cb1abbda511
8078438: Interpreter should support conditional card marks (UseCondCardMark) on x86 and aarch64
shade
parents:
31129
diff
changeset
|
5360 |
} else { |
2cb1abbda511
8078438: Interpreter should support conditional card marks (UseCondCardMark) on x86 and aarch64
shade
parents:
31129
diff
changeset
|
5361 |
movb(card_addr, dirty); |
14626 | 5362 |
} |
5363 |
} |
|
5364 |
||
5365 |
void MacroAssembler::subptr(Register dst, int32_t imm32) { |
|
5366 |
LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32)); |
|
5367 |
} |
|
5368 |
||
5369 |
// Force generation of a 4 byte immediate value even if it fits into 8bit |
|
5370 |
void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) { |
|
5371 |
LP64_ONLY(subq_imm32(dst, imm32)) NOT_LP64(subl_imm32(dst, imm32)); |
|
5372 |
} |
|
5373 |
||
5374 |
void MacroAssembler::subptr(Register dst, Register src) { |
|
5375 |
LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src)); |
|
5376 |
} |
|
5377 |
||
5378 |
// C++ bool manipulation |
|
5379 |
void MacroAssembler::testbool(Register dst) { |
|
5380 |
if(sizeof(bool) == 1) |
|
5381 |
testb(dst, 0xff); |
|
5382 |
else if(sizeof(bool) == 2) { |
|
5383 |
// testw implementation needed for two byte bools |
|
5384 |
ShouldNotReachHere(); |
|
5385 |
} else if(sizeof(bool) == 4) |
|
5386 |
testl(dst, dst); |
|
5387 |
else |
|
5388 |
// unsupported |
|
5389 |
ShouldNotReachHere(); |
|
5390 |
} |
|
5391 |
||
5392 |
void MacroAssembler::testptr(Register dst, Register src) { |
|
5393 |
LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src)); |
|
5394 |
} |
|
5395 |
||
5396 |
// Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes. |
|
5397 |
void MacroAssembler::tlab_allocate(Register obj, |
|
5398 |
Register var_size_in_bytes, |
|
5399 |
int con_size_in_bytes, |
|
5400 |
Register t1, |
|
5401 |
Register t2, |
|
5402 |
Label& slow_case) { |
|
5403 |
assert_different_registers(obj, t1, t2); |
|
5404 |
assert_different_registers(obj, var_size_in_bytes, t1); |
|
5405 |
Register end = t2; |
|
5406 |
Register thread = NOT_LP64(t1) LP64_ONLY(r15_thread); |
|
5407 |
||
5408 |
verify_tlab(); |
|
5409 |
||
5410 |
NOT_LP64(get_thread(thread)); |
|
5411 |
||
5412 |
movptr(obj, Address(thread, JavaThread::tlab_top_offset())); |
|
5413 |
if (var_size_in_bytes == noreg) { |
|
5414 |
lea(end, Address(obj, con_size_in_bytes)); |
|
5415 |
} else { |
|
5416 |
lea(end, Address(obj, var_size_in_bytes, Address::times_1)); |
|
5417 |
} |
|
5418 |
cmpptr(end, Address(thread, JavaThread::tlab_end_offset())); |
|
5419 |
jcc(Assembler::above, slow_case); |
|
5420 |
||
5421 |
// update the tlab top pointer |
|
5422 |
movptr(Address(thread, JavaThread::tlab_top_offset()), end); |
|
5423 |
||
5424 |
// recover var_size_in_bytes if necessary |
|
5425 |
if (var_size_in_bytes == end) { |
|
5426 |
subptr(var_size_in_bytes, obj); |
|
5427 |
} |
|
5428 |
verify_tlab(); |
|
5429 |
} |
|
5430 |
||
5431 |
// Preserves rbx, and rdx. |
|
5432 |
Register MacroAssembler::tlab_refill(Label& retry, |
|
5433 |
Label& try_eden, |
|
5434 |
Label& slow_case) { |
|
5435 |
Register top = rax; |
|
35548
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5436 |
Register t1 = rcx; // object size |
14626 | 5437 |
Register t2 = rsi; |
5438 |
Register thread_reg = NOT_LP64(rdi) LP64_ONLY(r15_thread); |
|
5439 |
assert_different_registers(top, thread_reg, t1, t2, /* preserve: */ rbx, rdx); |
|
5440 |
Label do_refill, discard_tlab; |
|
5441 |
||
27625 | 5442 |
if (!Universe::heap()->supports_inline_contig_alloc()) { |
14626 | 5443 |
// No allocation in the shared eden. |
5444 |
jmp(slow_case); |
|
5445 |
} |
|
5446 |
||
5447 |
NOT_LP64(get_thread(thread_reg)); |
|
5448 |
||
5449 |
movptr(top, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset()))); |
|
5450 |
movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset()))); |
|
5451 |
||
5452 |
// calculate amount of free space |
|
5453 |
subptr(t1, top); |
|
5454 |
shrptr(t1, LogHeapWordSize); |
|
5455 |
||
5456 |
// Retain tlab and allocate object in shared space if |
|
5457 |
// the amount free in the tlab is too large to discard. |
|
5458 |
cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_refill_waste_limit_offset()))); |
|
5459 |
jcc(Assembler::lessEqual, discard_tlab); |
|
5460 |
||
5461 |
// Retain |
|
5462 |
// %%% yuck as movptr... |
|
5463 |
movptr(t2, (int32_t) ThreadLocalAllocBuffer::refill_waste_limit_increment()); |
|
5464 |
addptr(Address(thread_reg, in_bytes(JavaThread::tlab_refill_waste_limit_offset())), t2); |
|
5465 |
if (TLABStats) { |
|
5466 |
// increment number of slow_allocations |
|
5467 |
addl(Address(thread_reg, in_bytes(JavaThread::tlab_slow_allocations_offset())), 1); |
|
5468 |
} |
|
5469 |
jmp(try_eden); |
|
5470 |
||
5471 |
bind(discard_tlab); |
|
5472 |
if (TLABStats) { |
|
5473 |
// increment number of refills |
|
5474 |
addl(Address(thread_reg, in_bytes(JavaThread::tlab_number_of_refills_offset())), 1); |
|
5475 |
// accumulate wastage -- t1 is amount free in tlab |
|
5476 |
addl(Address(thread_reg, in_bytes(JavaThread::tlab_fast_refill_waste_offset())), t1); |
|
5477 |
} |
|
5478 |
||
5479 |
// if tlab is currently allocated (top or end != null) then |
|
5480 |
// fill [top, end + alignment_reserve) with array object |
|
5481 |
testptr(top, top); |
|
5482 |
jcc(Assembler::zero, do_refill); |
|
5483 |
||
5484 |
// set up the mark word |
|
5485 |
movptr(Address(top, oopDesc::mark_offset_in_bytes()), (intptr_t)markOopDesc::prototype()->copy_set_hash(0x2)); |
|
5486 |
// set the length to the remaining space |
|
5487 |
subptr(t1, typeArrayOopDesc::header_size(T_INT)); |
|
5488 |
addptr(t1, (int32_t)ThreadLocalAllocBuffer::alignment_reserve()); |
|
5489 |
shlptr(t1, log2_intptr(HeapWordSize/sizeof(jint))); |
|
5490 |
movl(Address(top, arrayOopDesc::length_offset_in_bytes()), t1); |
|
5491 |
// set klass to intArrayKlass |
|
5492 |
// dubious reloc why not an oop reloc? |
|
5493 |
movptr(t1, ExternalAddress((address)Universe::intArrayKlassObj_addr())); |
|
5494 |
// store klass last. concurrent gcs assumes klass length is valid if |
|
5495 |
// klass field is not null. |
|
5496 |
store_klass(top, t1); |
|
5497 |
||
5498 |
movptr(t1, top); |
|
5499 |
subptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset()))); |
|
5500 |
incr_allocated_bytes(thread_reg, t1, 0); |
|
5501 |
||
5502 |
// refill the tlab with an eden allocation |
|
5503 |
bind(do_refill); |
|
5504 |
movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_size_offset()))); |
|
5505 |
shlptr(t1, LogHeapWordSize); |
|
5506 |
// allocate new tlab, address returned in top |
|
5507 |
eden_allocate(top, t1, 0, t2, slow_case); |
|
5508 |
||
5509 |
// Check that t1 was preserved in eden_allocate. |
|
5510 |
#ifdef ASSERT |
|
5511 |
if (UseTLAB) { |
|
5512 |
Label ok; |
|
5513 |
Register tsize = rsi; |
|
5514 |
assert_different_registers(tsize, thread_reg, t1); |
|
5515 |
push(tsize); |
|
5516 |
movptr(tsize, Address(thread_reg, in_bytes(JavaThread::tlab_size_offset()))); |
|
5517 |
shlptr(tsize, LogHeapWordSize); |
|
5518 |
cmpptr(t1, tsize); |
|
5519 |
jcc(Assembler::equal, ok); |
|
5520 |
STOP("assert(t1 != tlab size)"); |
|
5521 |
should_not_reach_here(); |
|
5522 |
||
5523 |
bind(ok); |
|
5524 |
pop(tsize); |
|
5525 |
} |
|
5526 |
#endif |
|
5527 |
movptr(Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())), top); |
|
5528 |
movptr(Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())), top); |
|
5529 |
addptr(top, t1); |
|
5530 |
subptr(top, (int32_t)ThreadLocalAllocBuffer::alignment_reserve_in_bytes()); |
|
5531 |
movptr(Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())), top); |
|
35548
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5532 |
|
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5533 |
if (ZeroTLAB) { |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5534 |
// This is a fast TLAB refill, therefore the GC is not notified of it. |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5535 |
// So compiled code must fill the new TLAB with zeroes. |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5536 |
movptr(top, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset()))); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5537 |
zero_memory(top, t1, 0, t2); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5538 |
} |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5539 |
|
14626 | 5540 |
verify_tlab(); |
5541 |
jmp(retry); |
|
5542 |
||
5543 |
return thread_reg; // for use by caller |
|
5544 |
} |
|
5545 |
||
35548
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5546 |
// Preserves the contents of address, destroys the contents length_in_bytes and temp. |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5547 |
void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) { |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5548 |
assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different"); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5549 |
assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord"); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5550 |
Label done; |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5551 |
|
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5552 |
testptr(length_in_bytes, length_in_bytes); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5553 |
jcc(Assembler::zero, done); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5554 |
|
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5555 |
// initialize topmost word, divide index by 2, check if odd and test if zero |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5556 |
// note: for the remaining code to work, index must be a multiple of BytesPerWord |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5557 |
#ifdef ASSERT |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5558 |
{ |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5559 |
Label L; |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5560 |
testptr(length_in_bytes, BytesPerWord - 1); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5561 |
jcc(Assembler::zero, L); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5562 |
stop("length must be a multiple of BytesPerWord"); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5563 |
bind(L); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5564 |
} |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5565 |
#endif |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5566 |
Register index = length_in_bytes; |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5567 |
xorptr(temp, temp); // use _zero reg to clear memory (shorter code) |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5568 |
if (UseIncDec) { |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5569 |
shrptr(index, 3); // divide by 8/16 and set carry flag if bit 2 was set |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5570 |
} else { |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5571 |
shrptr(index, 2); // use 2 instructions to avoid partial flag stall |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5572 |
shrptr(index, 1); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5573 |
} |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5574 |
#ifndef _LP64 |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5575 |
// index could have not been a multiple of 8 (i.e., bit 2 was set) |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5576 |
{ |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5577 |
Label even; |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5578 |
// note: if index was a multiple of 8, then it cannot |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5579 |
// be 0 now otherwise it must have been 0 before |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5580 |
// => if it is even, we don't need to check for 0 again |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5581 |
jcc(Assembler::carryClear, even); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5582 |
// clear topmost word (no jump would be needed if conditional assignment worked here) |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5583 |
movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5584 |
// index could be 0 now, must check again |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5585 |
jcc(Assembler::zero, done); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5586 |
bind(even); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5587 |
} |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5588 |
#endif // !_LP64 |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5589 |
// initialize remaining object fields: index is a multiple of 2 now |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5590 |
{ |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5591 |
Label loop; |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5592 |
bind(loop); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5593 |
movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5594 |
NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);) |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5595 |
decrement(index); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5596 |
jcc(Assembler::notZero, loop); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5597 |
} |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5598 |
|
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5599 |
bind(done); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5600 |
} |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
5601 |
|
14626 | 5602 |
void MacroAssembler::incr_allocated_bytes(Register thread, |
5603 |
Register var_size_in_bytes, |
|
5604 |
int con_size_in_bytes, |
|
5605 |
Register t1) { |
|
5606 |
if (!thread->is_valid()) { |
|
5607 |
#ifdef _LP64 |
|
5608 |
thread = r15_thread; |
|
5609 |
#else |
|
5610 |
assert(t1->is_valid(), "need temp reg"); |
|
5611 |
thread = t1; |
|
5612 |
get_thread(thread); |
|
5613 |
#endif |
|
5614 |
} |
|
5615 |
||
5616 |
#ifdef _LP64 |
|
5617 |
if (var_size_in_bytes->is_valid()) { |
|
5618 |
addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes); |
|
5619 |
} else { |
|
5620 |
addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes); |
|
5621 |
} |
|
5622 |
#else |
|
5623 |
if (var_size_in_bytes->is_valid()) { |
|
5624 |
addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes); |
|
5625 |
} else { |
|
5626 |
addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes); |
|
5627 |
} |
|
5628 |
adcl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())+4), 0); |
|
5629 |
#endif |
|
5630 |
} |
|
5631 |
||
5632 |
void MacroAssembler::fp_runtime_fallback(address runtime_entry, int nb_args, int num_fpu_regs_in_use) { |
|
5633 |
pusha(); |
|
5634 |
||
5635 |
// if we are coming from c1, xmm registers may be live |
|
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5636 |
int num_xmm_regs = LP64_ONLY(16) NOT_LP64(8); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5637 |
if (UseAVX > 2) { |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5638 |
num_xmm_regs = LP64_ONLY(32) NOT_LP64(8); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5639 |
} |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5640 |
|
14626 | 5641 |
if (UseSSE == 1) { |
5642 |
subptr(rsp, sizeof(jdouble)*8); |
|
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5643 |
for (int n = 0; n < 8; n++) { |
34162 | 5644 |
movflt(Address(rsp, n*sizeof(jdouble)), as_XMMRegister(n)); |
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5645 |
} |
14626 | 5646 |
} else if (UseSSE >= 2) { |
30624 | 5647 |
if (UseAVX > 2) { |
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5648 |
push(rbx); |
30624 | 5649 |
movl(rbx, 0xffff); |
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5650 |
kmovwl(k1, rbx); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5651 |
pop(rbx); |
30624 | 5652 |
} |
14626 | 5653 |
#ifdef COMPILER2 |
5654 |
if (MaxVectorSize > 16) { |
|
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5655 |
if(UseAVX > 2) { |
34162 | 5656 |
// Save upper half of ZMM registers |
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5657 |
subptr(rsp, 32*num_xmm_regs); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5658 |
for (int n = 0; n < num_xmm_regs; n++) { |
36561
b18243f4d955
8151002: Make Assembler methods vextract and vinsert match actual instructions
mikael
parents:
36554
diff
changeset
|
5659 |
vextractf64x4_high(Address(rsp, n*32), as_XMMRegister(n)); |
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5660 |
} |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5661 |
} |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5662 |
assert(UseAVX > 0, "256 bit vectors are supported only with AVX"); |
34162 | 5663 |
// Save upper half of YMM registers |
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5664 |
subptr(rsp, 16*num_xmm_regs); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5665 |
for (int n = 0; n < num_xmm_regs; n++) { |
36561
b18243f4d955
8151002: Make Assembler methods vextract and vinsert match actual instructions
mikael
parents:
36554
diff
changeset
|
5666 |
vextractf128_high(Address(rsp, n*16), as_XMMRegister(n)); |
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5667 |
} |
14626 | 5668 |
} |
5669 |
#endif |
|
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5670 |
// Save whole 128bit (16 bytes) XMM registers |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5671 |
subptr(rsp, 16*num_xmm_regs); |
14626 | 5672 |
#ifdef _LP64 |
34162 | 5673 |
if (VM_Version::supports_evex()) { |
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5674 |
for (int n = 0; n < num_xmm_regs; n++) { |
36561
b18243f4d955
8151002: Make Assembler methods vextract and vinsert match actual instructions
mikael
parents:
36554
diff
changeset
|
5675 |
vextractf32x4(Address(rsp, n*16), as_XMMRegister(n), 0); |
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5676 |
} |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5677 |
} else { |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5678 |
for (int n = 0; n < num_xmm_regs; n++) { |
34162 | 5679 |
movdqu(Address(rsp, n*16), as_XMMRegister(n)); |
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5680 |
} |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5681 |
} |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5682 |
#else |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5683 |
for (int n = 0; n < num_xmm_regs; n++) { |
34162 | 5684 |
movdqu(Address(rsp, n*16), as_XMMRegister(n)); |
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5685 |
} |
14626 | 5686 |
#endif |
5687 |
} |
|
5688 |
||
5689 |
// Preserve registers across runtime call |
|
5690 |
int incoming_argument_and_return_value_offset = -1; |
|
5691 |
if (num_fpu_regs_in_use > 1) { |
|
5692 |
// Must preserve all other FPU regs (could alternatively convert |
|
5693 |
// SharedRuntime::dsin, dcos etc. into assembly routines known not to trash |
|
5694 |
// FPU state, but can not trust C compiler) |
|
5695 |
NEEDS_CLEANUP; |
|
5696 |
// NOTE that in this case we also push the incoming argument(s) to |
|
5697 |
// the stack and restore it later; we also use this stack slot to |
|
5698 |
// hold the return value from dsin, dcos etc. |
|
5699 |
for (int i = 0; i < num_fpu_regs_in_use; i++) { |
|
5700 |
subptr(rsp, sizeof(jdouble)); |
|
5701 |
fstp_d(Address(rsp, 0)); |
|
5702 |
} |
|
5703 |
incoming_argument_and_return_value_offset = sizeof(jdouble)*(num_fpu_regs_in_use-1); |
|
5704 |
for (int i = nb_args-1; i >= 0; i--) { |
|
5705 |
fld_d(Address(rsp, incoming_argument_and_return_value_offset-i*sizeof(jdouble))); |
|
5706 |
} |
|
5707 |
} |
|
5708 |
||
5709 |
subptr(rsp, nb_args*sizeof(jdouble)); |
|
5710 |
for (int i = 0; i < nb_args; i++) { |
|
5711 |
fstp_d(Address(rsp, i*sizeof(jdouble))); |
|
5712 |
} |
|
5713 |
||
5714 |
#ifdef _LP64 |
|
5715 |
if (nb_args > 0) { |
|
5716 |
movdbl(xmm0, Address(rsp, 0)); |
|
5717 |
} |
|
5718 |
if (nb_args > 1) { |
|
5719 |
movdbl(xmm1, Address(rsp, sizeof(jdouble))); |
|
5720 |
} |
|
5721 |
assert(nb_args <= 2, "unsupported number of args"); |
|
5722 |
#endif // _LP64 |
|
5723 |
||
5724 |
// NOTE: we must not use call_VM_leaf here because that requires a |
|
5725 |
// complete interpreter frame in debug mode -- same bug as 4387334 |
|
5726 |
// MacroAssembler::call_VM_leaf_base is perfectly safe and will |
|
5727 |
// do proper 64bit abi |
|
5728 |
||
5729 |
NEEDS_CLEANUP; |
|
5730 |
// Need to add stack banging before this runtime call if it needs to |
|
5731 |
// be taken; however, there is no generic stack banging routine at |
|
5732 |
// the MacroAssembler level |
|
5733 |
||
5734 |
MacroAssembler::call_VM_leaf_base(runtime_entry, 0); |
|
5735 |
||
5736 |
#ifdef _LP64 |
|
5737 |
movsd(Address(rsp, 0), xmm0); |
|
5738 |
fld_d(Address(rsp, 0)); |
|
5739 |
#endif // _LP64 |
|
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5740 |
addptr(rsp, sizeof(jdouble)*nb_args); |
14626 | 5741 |
if (num_fpu_regs_in_use > 1) { |
5742 |
// Must save return value to stack and then restore entire FPU |
|
5743 |
// stack except incoming arguments |
|
5744 |
fstp_d(Address(rsp, incoming_argument_and_return_value_offset)); |
|
5745 |
for (int i = 0; i < num_fpu_regs_in_use - nb_args; i++) { |
|
5746 |
fld_d(Address(rsp, 0)); |
|
5747 |
addptr(rsp, sizeof(jdouble)); |
|
5748 |
} |
|
5749 |
fld_d(Address(rsp, (nb_args-1)*sizeof(jdouble))); |
|
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5750 |
addptr(rsp, sizeof(jdouble)*nb_args); |
14626 | 5751 |
} |
5752 |
||
5753 |
if (UseSSE == 1) { |
|
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5754 |
for (int n = 0; n < 8; n++) { |
34162 | 5755 |
movflt(as_XMMRegister(n), Address(rsp, n*sizeof(jdouble))); |
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5756 |
} |
14626 | 5757 |
addptr(rsp, sizeof(jdouble)*8); |
5758 |
} else if (UseSSE >= 2) { |
|
34162 | 5759 |
// Restore whole 128bit (16 bytes) XMM registers |
14626 | 5760 |
#ifdef _LP64 |
34162 | 5761 |
if (VM_Version::supports_evex()) { |
5762 |
for (int n = 0; n < num_xmm_regs; n++) { |
|
36561
b18243f4d955
8151002: Make Assembler methods vextract and vinsert match actual instructions
mikael
parents:
36554
diff
changeset
|
5763 |
vinsertf32x4(as_XMMRegister(n), as_XMMRegister(n), Address(rsp, n*16), 0); |
34162 | 5764 |
} |
5765 |
} else { |
|
5766 |
for (int n = 0; n < num_xmm_regs; n++) { |
|
5767 |
movdqu(as_XMMRegister(n), Address(rsp, n*16)); |
|
5768 |
} |
|
5769 |
} |
|
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5770 |
#else |
34162 | 5771 |
for (int n = 0; n < num_xmm_regs; n++) { |
5772 |
movdqu(as_XMMRegister(n), Address(rsp, n*16)); |
|
5773 |
} |
|
14626 | 5774 |
#endif |
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5775 |
addptr(rsp, 16*num_xmm_regs); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5776 |
|
14626 | 5777 |
#ifdef COMPILER2 |
5778 |
if (MaxVectorSize > 16) { |
|
34162 | 5779 |
// Restore upper half of YMM registers. |
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5780 |
for (int n = 0; n < num_xmm_regs; n++) { |
36561
b18243f4d955
8151002: Make Assembler methods vextract and vinsert match actual instructions
mikael
parents:
36554
diff
changeset
|
5781 |
vinsertf128_high(as_XMMRegister(n), Address(rsp, n*16)); |
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5782 |
} |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5783 |
addptr(rsp, 16*num_xmm_regs); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5784 |
if(UseAVX > 2) { |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5785 |
for (int n = 0; n < num_xmm_regs; n++) { |
36561
b18243f4d955
8151002: Make Assembler methods vextract and vinsert match actual instructions
mikael
parents:
36554
diff
changeset
|
5786 |
vinsertf64x4_high(as_XMMRegister(n), Address(rsp, n*32)); |
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5787 |
} |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5788 |
addptr(rsp, 32*num_xmm_regs); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
5789 |
} |
14626 | 5790 |
} |
5791 |
#endif |
|
5792 |
} |
|
5793 |
popa(); |
|
5794 |
} |
|
5795 |
||
5796 |
static const double pi_4 = 0.7853981633974483; |
|
5797 |
||
5798 |
void MacroAssembler::trigfunc(char trig, int num_fpu_regs_in_use) { |
|
5799 |
// A hand-coded argument reduction for values in fabs(pi/4, pi/2) |
|
5800 |
// was attempted in this code; unfortunately it appears that the |
|
5801 |
// switch to 80-bit precision and back causes this to be |
|
5802 |
// unprofitable compared with simply performing a runtime call if |
|
5803 |
// the argument is out of the (-pi/4, pi/4) range. |
|
5804 |
||
5805 |
Register tmp = noreg; |
|
5806 |
if (!VM_Version::supports_cmov()) { |
|
5807 |
// fcmp needs a temporary so preserve rbx, |
|
5808 |
tmp = rbx; |
|
5809 |
push(tmp); |
|
5810 |
} |
|
5811 |
||
5812 |
Label slow_case, done; |
|
35540
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
5813 |
if (trig == 't') { |
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
5814 |
ExternalAddress pi4_adr = (address)&pi_4; |
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
5815 |
if (reachable(pi4_adr)) { |
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
5816 |
// x ?<= pi/4 |
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
5817 |
fld_d(pi4_adr); |
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
5818 |
fld_s(1); // Stack: X PI/4 X |
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
5819 |
fabs(); // Stack: |X| PI/4 X |
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
5820 |
fcmp(tmp); |
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
5821 |
jcc(Assembler::above, slow_case); |
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
5822 |
|
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
5823 |
// fastest case: -pi/4 <= x <= pi/4 |
14626 | 5824 |
ftan(); |
35540
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
5825 |
|
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
5826 |
jmp(done); |
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
5827 |
} |
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
5828 |
} |
14626 | 5829 |
// slow case: runtime call |
5830 |
bind(slow_case); |
|
5831 |
||
5832 |
switch(trig) { |
|
5833 |
case 's': |
|
5834 |
{ |
|
5835 |
fp_runtime_fallback(CAST_FROM_FN_PTR(address, SharedRuntime::dsin), 1, num_fpu_regs_in_use); |
|
5836 |
} |
|
5837 |
break; |
|
5838 |
case 'c': |
|
5839 |
{ |
|
5840 |
fp_runtime_fallback(CAST_FROM_FN_PTR(address, SharedRuntime::dcos), 1, num_fpu_regs_in_use); |
|
5841 |
} |
|
5842 |
break; |
|
5843 |
case 't': |
|
5844 |
{ |
|
5845 |
fp_runtime_fallback(CAST_FROM_FN_PTR(address, SharedRuntime::dtan), 1, num_fpu_regs_in_use); |
|
5846 |
} |
|
5847 |
break; |
|
5848 |
default: |
|
5849 |
assert(false, "bad intrinsic"); |
|
5850 |
break; |
|
5851 |
} |
|
5852 |
||
5853 |
// Come here with result in F-TOS |
|
5854 |
bind(done); |
|
5855 |
||
5856 |
if (tmp != noreg) { |
|
5857 |
pop(tmp); |
|
5858 |
} |
|
5859 |
} |
|
5860 |
||
5861 |
// Look up the method for a megamorphic invokeinterface call. |
|
5862 |
// The target method is determined by <intf_klass, itable_index>. |
|
5863 |
// The receiver klass is in recv_klass. |
|
5864 |
// On success, the result will be in method_result, and execution falls through. |
|
5865 |
// On failure, execution transfers to the given label. |
|
5866 |
void MacroAssembler::lookup_interface_method(Register recv_klass, |
|
5867 |
Register intf_klass, |
|
5868 |
RegisterOrConstant itable_index, |
|
5869 |
Register method_result, |
|
5870 |
Register scan_temp, |
|
5871 |
Label& L_no_such_interface) { |
|
5872 |
assert_different_registers(recv_klass, intf_klass, method_result, scan_temp); |
|
5873 |
assert(itable_index.is_constant() || itable_index.as_register() == method_result, |
|
5874 |
"caller must use same register for non-constant itable index as for method"); |
|
5875 |
||
5876 |
// Compute start of first itableOffsetEntry (which is at the end of the vtable) |
|
35899 | 5877 |
int vtable_base = in_bytes(Klass::vtable_start_offset()); |
14626 | 5878 |
int itentry_off = itableMethodEntry::method_offset_in_bytes(); |
5879 |
int scan_step = itableOffsetEntry::size() * wordSize; |
|
35871
607bf949dfb3
8147461: Use byte offsets for vtable start and vtable length offsets
mgerdin
parents:
35847
diff
changeset
|
5880 |
int vte_size = vtableEntry::size_in_bytes(); |
14626 | 5881 |
Address::ScaleFactor times_vte_scale = Address::times_ptr; |
5882 |
assert(vte_size == wordSize, "else adjust times_vte_scale"); |
|
5883 |
||
35899 | 5884 |
movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset())); |
14626 | 5885 |
|
5886 |
// %%% Could store the aligned, prescaled offset in the klassoop. |
|
5887 |
lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base)); |
|
5888 |
||
5889 |
// Adjust recv_klass by scaled itable_index, so we can free itable_index. |
|
5890 |
assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below"); |
|
5891 |
lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off)); |
|
5892 |
||
5893 |
// for (scan = klass->itable(); scan->interface() != NULL; scan += scan_step) { |
|
5894 |
// if (scan->interface() == intf) { |
|
5895 |
// result = (klass + scan->offset() + itable_index); |
|
5896 |
// } |
|
5897 |
// } |
|
5898 |
Label search, found_method; |
|
5899 |
||
5900 |
for (int peel = 1; peel >= 0; peel--) { |
|
5901 |
movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset_in_bytes())); |
|
5902 |
cmpptr(intf_klass, method_result); |
|
5903 |
||
5904 |
if (peel) { |
|
5905 |
jccb(Assembler::equal, found_method); |
|
5906 |
} else { |
|
5907 |
jccb(Assembler::notEqual, search); |
|
5908 |
// (invert the test to fall through to found_method...) |
|
5909 |
} |
|
5910 |
||
5911 |
if (!peel) break; |
|
5912 |
||
5913 |
bind(search); |
|
5914 |
||
5915 |
// Check that the previous entry is non-null. A null entry means that |
|
5916 |
// the receiver class doesn't implement the interface, and wasn't the |
|
5917 |
// same as when the caller was compiled. |
|
5918 |
testptr(method_result, method_result); |
|
5919 |
jcc(Assembler::zero, L_no_such_interface); |
|
5920 |
addptr(scan_temp, scan_step); |
|
5921 |
} |
|
5922 |
||
5923 |
bind(found_method); |
|
5924 |
||
5925 |
// Got a hit. |
|
5926 |
movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset_in_bytes())); |
|
5927 |
movptr(method_result, Address(recv_klass, scan_temp, Address::times_1)); |
|
5928 |
} |
|
5929 |
||
5930 |
||
5931 |
// virtual method calling |
|
5932 |
void MacroAssembler::lookup_virtual_method(Register recv_klass, |
|
5933 |
RegisterOrConstant vtable_index, |
|
5934 |
Register method_result) { |
|
35899 | 5935 |
const int base = in_bytes(Klass::vtable_start_offset()); |
14626 | 5936 |
assert(vtableEntry::size() * wordSize == wordSize, "else adjust the scaling in the code below"); |
5937 |
Address vtable_entry_addr(recv_klass, |
|
5938 |
vtable_index, Address::times_ptr, |
|
5939 |
base + vtableEntry::method_offset_in_bytes()); |
|
5940 |
movptr(method_result, vtable_entry_addr); |
|
5941 |
} |
|
5942 |
||
5943 |
||
5944 |
void MacroAssembler::check_klass_subtype(Register sub_klass, |
|
5945 |
Register super_klass, |
|
5946 |
Register temp_reg, |
|
5947 |
Label& L_success) { |
|
5948 |
Label L_failure; |
|
5949 |
check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg, &L_success, &L_failure, NULL); |
|
5950 |
check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, NULL); |
|
5951 |
bind(L_failure); |
|
5952 |
} |
|
5953 |
||
5954 |
||
5955 |
void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass, |
|
5956 |
Register super_klass, |
|
5957 |
Register temp_reg, |
|
5958 |
Label* L_success, |
|
5959 |
Label* L_failure, |
|
5960 |
Label* L_slow_path, |
|
5961 |
RegisterOrConstant super_check_offset) { |
|
5962 |
assert_different_registers(sub_klass, super_klass, temp_reg); |
|
5963 |
bool must_load_sco = (super_check_offset.constant_or_zero() == -1); |
|
5964 |
if (super_check_offset.is_register()) { |
|
5965 |
assert_different_registers(sub_klass, super_klass, |
|
5966 |
super_check_offset.as_register()); |
|
5967 |
} else if (must_load_sco) { |
|
5968 |
assert(temp_reg != noreg, "supply either a temp or a register offset"); |
|
5969 |
} |
|
5970 |
||
5971 |
Label L_fallthrough; |
|
5972 |
int label_nulls = 0; |
|
5973 |
if (L_success == NULL) { L_success = &L_fallthrough; label_nulls++; } |
|
5974 |
if (L_failure == NULL) { L_failure = &L_fallthrough; label_nulls++; } |
|
5975 |
if (L_slow_path == NULL) { L_slow_path = &L_fallthrough; label_nulls++; } |
|
5976 |
assert(label_nulls <= 1, "at most one NULL in the batch"); |
|
5977 |
||
5978 |
int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); |
|
5979 |
int sco_offset = in_bytes(Klass::super_check_offset_offset()); |
|
5980 |
Address super_check_offset_addr(super_klass, sco_offset); |
|
5981 |
||
5982 |
// Hacked jcc, which "knows" that L_fallthrough, at least, is in |
|
5983 |
// range of a jccb. If this routine grows larger, reconsider at |
|
5984 |
// least some of these. |
|
5985 |
#define local_jcc(assembler_cond, label) \ |
|
5986 |
if (&(label) == &L_fallthrough) jccb(assembler_cond, label); \ |
|
5987 |
else jcc( assembler_cond, label) /*omit semi*/ |
|
5988 |
||
5989 |
// Hacked jmp, which may only be used just before L_fallthrough. |
|
5990 |
#define final_jmp(label) \ |
|
5991 |
if (&(label) == &L_fallthrough) { /*do nothing*/ } \ |
|
5992 |
else jmp(label) /*omit semi*/ |
|
5993 |
||
5994 |
// If the pointers are equal, we are done (e.g., String[] elements). |
|
5995 |
// This self-check enables sharing of secondary supertype arrays among |
|
5996 |
// non-primary types such as array-of-interface. Otherwise, each such |
|
5997 |
// type would need its own customized SSA. |
|
5998 |
// We move this check to the front of the fast path because many |
|
5999 |
// type checks are in fact trivially successful in this manner, |
|
6000 |
// so we get a nicely predicted branch right at the start of the check. |
|
6001 |
cmpptr(sub_klass, super_klass); |
|
6002 |
local_jcc(Assembler::equal, *L_success); |
|
6003 |
||
6004 |
// Check the supertype display: |
|
6005 |
if (must_load_sco) { |
|
6006 |
// Positive movl does right thing on LP64. |
|
6007 |
movl(temp_reg, super_check_offset_addr); |
|
6008 |
super_check_offset = RegisterOrConstant(temp_reg); |
|
6009 |
} |
|
6010 |
Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0); |
|
6011 |
cmpptr(super_klass, super_check_addr); // load displayed supertype |
|
6012 |
||
6013 |
// This check has worked decisively for primary supers. |
|
6014 |
// Secondary supers are sought in the super_cache ('super_cache_addr'). |
|
6015 |
// (Secondary supers are interfaces and very deeply nested subtypes.) |
|
6016 |
// This works in the same check above because of a tricky aliasing |
|
6017 |
// between the super_cache and the primary super display elements. |
|
6018 |
// (The 'super_check_addr' can address either, as the case requires.) |
|
6019 |
// Note that the cache is updated below if it does not help us find |
|
6020 |
// what we need immediately. |
|
6021 |
// So if it was a primary super, we can just fail immediately. |
|
6022 |
// Otherwise, it's the slow path for us (no success at this point). |
|
6023 |
||
6024 |
if (super_check_offset.is_register()) { |
|
6025 |
local_jcc(Assembler::equal, *L_success); |
|
6026 |
cmpl(super_check_offset.as_register(), sc_offset); |
|
6027 |
if (L_failure == &L_fallthrough) { |
|
6028 |
local_jcc(Assembler::equal, *L_slow_path); |
|
6029 |
} else { |
|
6030 |
local_jcc(Assembler::notEqual, *L_failure); |
|
6031 |
final_jmp(*L_slow_path); |
|
6032 |
} |
|
6033 |
} else if (super_check_offset.as_constant() == sc_offset) { |
|
6034 |
// Need a slow path; fast failure is impossible. |
|
6035 |
if (L_slow_path == &L_fallthrough) { |
|
6036 |
local_jcc(Assembler::equal, *L_success); |
|
6037 |
} else { |
|
6038 |
local_jcc(Assembler::notEqual, *L_slow_path); |
|
6039 |
final_jmp(*L_success); |
|
6040 |
} |
|
6041 |
} else { |
|
6042 |
// No slow path; it's a fast decision. |
|
6043 |
if (L_failure == &L_fallthrough) { |
|
6044 |
local_jcc(Assembler::equal, *L_success); |
|
6045 |
} else { |
|
6046 |
local_jcc(Assembler::notEqual, *L_failure); |
|
6047 |
final_jmp(*L_success); |
|
6048 |
} |
|
6049 |
} |
|
6050 |
||
6051 |
bind(L_fallthrough); |
|
6052 |
||
6053 |
#undef local_jcc |
|
6054 |
#undef final_jmp |
|
6055 |
} |
|
6056 |
||
6057 |
||
6058 |
void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass, |
|
6059 |
Register super_klass, |
|
6060 |
Register temp_reg, |
|
6061 |
Register temp2_reg, |
|
6062 |
Label* L_success, |
|
6063 |
Label* L_failure, |
|
6064 |
bool set_cond_codes) { |
|
6065 |
assert_different_registers(sub_klass, super_klass, temp_reg); |
|
6066 |
if (temp2_reg != noreg) |
|
6067 |
assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg); |
|
6068 |
#define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg) |
|
6069 |
||
6070 |
Label L_fallthrough; |
|
6071 |
int label_nulls = 0; |
|
6072 |
if (L_success == NULL) { L_success = &L_fallthrough; label_nulls++; } |
|
6073 |
if (L_failure == NULL) { L_failure = &L_fallthrough; label_nulls++; } |
|
6074 |
assert(label_nulls <= 1, "at most one NULL in the batch"); |
|
6075 |
||
6076 |
// a couple of useful fields in sub_klass: |
|
6077 |
int ss_offset = in_bytes(Klass::secondary_supers_offset()); |
|
6078 |
int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); |
|
6079 |
Address secondary_supers_addr(sub_klass, ss_offset); |
|
6080 |
Address super_cache_addr( sub_klass, sc_offset); |
|
6081 |
||
6082 |
// Do a linear scan of the secondary super-klass chain. |
|
6083 |
// This code is rarely used, so simplicity is a virtue here. |
|
6084 |
// The repne_scan instruction uses fixed registers, which we must spill. |
|
6085 |
// Don't worry too much about pre-existing connections with the input regs. |
|
6086 |
||
6087 |
assert(sub_klass != rax, "killed reg"); // killed by mov(rax, super) |
|
6088 |
assert(sub_klass != rcx, "killed reg"); // killed by lea(rcx, &pst_counter) |
|
6089 |
||
6090 |
// Get super_klass value into rax (even if it was in rdi or rcx). |
|
6091 |
bool pushed_rax = false, pushed_rcx = false, pushed_rdi = false; |
|
6092 |
if (super_klass != rax || UseCompressedOops) { |
|
6093 |
if (!IS_A_TEMP(rax)) { push(rax); pushed_rax = true; } |
|
6094 |
mov(rax, super_klass); |
|
6095 |
} |
|
6096 |
if (!IS_A_TEMP(rcx)) { push(rcx); pushed_rcx = true; } |
|
6097 |
if (!IS_A_TEMP(rdi)) { push(rdi); pushed_rdi = true; } |
|
6098 |
||
6099 |
#ifndef PRODUCT |
|
6100 |
int* pst_counter = &SharedRuntime::_partial_subtype_ctr; |
|
6101 |
ExternalAddress pst_counter_addr((address) pst_counter); |
|
6102 |
NOT_LP64( incrementl(pst_counter_addr) ); |
|
6103 |
LP64_ONLY( lea(rcx, pst_counter_addr) ); |
|
6104 |
LP64_ONLY( incrementl(Address(rcx, 0)) ); |
|
6105 |
#endif //PRODUCT |
|
6106 |
||
6107 |
// We will consult the secondary-super array. |
|
6108 |
movptr(rdi, secondary_supers_addr); |
|
6109 |
// Load the array length. (Positive movl does right thing on LP64.) |
|
6110 |
movl(rcx, Address(rdi, Array<Klass*>::length_offset_in_bytes())); |
|
6111 |
// Skip to start of data. |
|
6112 |
addptr(rdi, Array<Klass*>::base_offset_in_bytes()); |
|
6113 |
||
6114 |
// Scan RCX words at [RDI] for an occurrence of RAX. |
|
6115 |
// Set NZ/Z based on last compare. |
|
6116 |
// Z flag value will not be set by 'repne' if RCX == 0 since 'repne' does |
|
6117 |
// not change flags (only scas instruction which is repeated sets flags). |
|
6118 |
// Set Z = 0 (not equal) before 'repne' to indicate that class was not found. |
|
6119 |
||
6120 |
testptr(rax,rax); // Set Z = 0 |
|
6121 |
repne_scan(); |
|
6122 |
||
6123 |
// Unspill the temp. registers: |
|
6124 |
if (pushed_rdi) pop(rdi); |
|
6125 |
if (pushed_rcx) pop(rcx); |
|
6126 |
if (pushed_rax) pop(rax); |
|
6127 |
||
6128 |
if (set_cond_codes) { |
|
6129 |
// Special hack for the AD files: rdi is guaranteed non-zero. |
|
6130 |
assert(!pushed_rdi, "rdi must be left non-NULL"); |
|
6131 |
// Also, the condition codes are properly set Z/NZ on succeed/failure. |
|
6132 |
} |
|
6133 |
||
6134 |
if (L_failure == &L_fallthrough) |
|
6135 |
jccb(Assembler::notEqual, *L_failure); |
|
6136 |
else jcc(Assembler::notEqual, *L_failure); |
|
6137 |
||
6138 |
// Success. Cache the super we found and proceed in triumph. |
|
6139 |
movptr(super_cache_addr, super_klass); |
|
6140 |
||
6141 |
if (L_success != &L_fallthrough) { |
|
6142 |
jmp(*L_success); |
|
6143 |
} |
|
6144 |
||
6145 |
#undef IS_A_TEMP |
|
6146 |
||
6147 |
bind(L_fallthrough); |
|
6148 |
} |
|
6149 |
||
6150 |
||
6151 |
void MacroAssembler::cmov32(Condition cc, Register dst, Address src) { |
|
6152 |
if (VM_Version::supports_cmov()) { |
|
6153 |
cmovl(cc, dst, src); |
|
6154 |
} else { |
|
6155 |
Label L; |
|
6156 |
jccb(negate_condition(cc), L); |
|
6157 |
movl(dst, src); |
|
6158 |
bind(L); |
|
6159 |
} |
|
6160 |
} |
|
6161 |
||
6162 |
void MacroAssembler::cmov32(Condition cc, Register dst, Register src) { |
|
6163 |
if (VM_Version::supports_cmov()) { |
|
6164 |
cmovl(cc, dst, src); |
|
6165 |
} else { |
|
6166 |
Label L; |
|
6167 |
jccb(negate_condition(cc), L); |
|
6168 |
movl(dst, src); |
|
6169 |
bind(L); |
|
6170 |
} |
|
6171 |
} |
|
6172 |
||
6173 |
void MacroAssembler::verify_oop(Register reg, const char* s) { |
|
6174 |
if (!VerifyOops) return; |
|
6175 |
||
6176 |
// Pass register number to verify_oop_subroutine |
|
16368
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
6177 |
const char* b = NULL; |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
6178 |
{ |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
6179 |
ResourceMark rm; |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
6180 |
stringStream ss; |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
6181 |
ss.print("verify_oop: %s: %s", reg->name(), s); |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
6182 |
b = code_string(ss.as_string()); |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
6183 |
} |
14626 | 6184 |
BLOCK_COMMENT("verify_oop {"); |
6185 |
#ifdef _LP64 |
|
6186 |
push(rscratch1); // save r10, trashed by movptr() |
|
6187 |
#endif |
|
6188 |
push(rax); // save rax, |
|
6189 |
push(reg); // pass register argument |
|
6190 |
ExternalAddress buffer((address) b); |
|
6191 |
// avoid using pushptr, as it modifies scratch registers |
|
6192 |
// and our contract is not to modify anything |
|
6193 |
movptr(rax, buffer.addr()); |
|
6194 |
push(rax); |
|
6195 |
// call indirectly to solve generation ordering problem |
|
6196 |
movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address())); |
|
6197 |
call(rax); |
|
6198 |
// Caller pops the arguments (oop, message) and restores rax, r10 |
|
6199 |
BLOCK_COMMENT("} verify_oop"); |
|
6200 |
} |
|
6201 |
||
6202 |
||
6203 |
RegisterOrConstant MacroAssembler::delayed_value_impl(intptr_t* delayed_value_addr, |
|
6204 |
Register tmp, |
|
6205 |
int offset) { |
|
6206 |
intptr_t value = *delayed_value_addr; |
|
6207 |
if (value != 0) |
|
6208 |
return RegisterOrConstant(value + offset); |
|
6209 |
||
6210 |
// load indirectly to solve generation ordering problem |
|
6211 |
movptr(tmp, ExternalAddress((address) delayed_value_addr)); |
|
6212 |
||
6213 |
#ifdef ASSERT |
|
6214 |
{ Label L; |
|
6215 |
testptr(tmp, tmp); |
|
6216 |
if (WizardMode) { |
|
16368
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
6217 |
const char* buf = NULL; |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
6218 |
{ |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
6219 |
ResourceMark rm; |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
6220 |
stringStream ss; |
31592
43f48e165466
8081202: Hotspot compile warning: "Invalid suffix on literal; C++11 requires a space between literal and identifier"
bpittore
parents:
31369
diff
changeset
|
6221 |
ss.print("DelayedValue=" INTPTR_FORMAT, delayed_value_addr[1]); |
16368
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
6222 |
buf = code_string(ss.as_string()); |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
6223 |
} |
14626 | 6224 |
jcc(Assembler::notZero, L); |
6225 |
STOP(buf); |
|
6226 |
} else { |
|
6227 |
jccb(Assembler::notZero, L); |
|
6228 |
hlt(); |
|
6229 |
} |
|
6230 |
bind(L); |
|
6231 |
} |
|
6232 |
#endif |
|
6233 |
||
6234 |
if (offset != 0) |
|
6235 |
addptr(tmp, offset); |
|
6236 |
||
6237 |
return RegisterOrConstant(tmp); |
|
6238 |
} |
|
6239 |
||
6240 |
||
6241 |
Address MacroAssembler::argument_address(RegisterOrConstant arg_slot, |
|
6242 |
int extra_slot_offset) { |
|
6243 |
// cf. TemplateTable::prepare_invoke(), if (load_receiver). |
|
6244 |
int stackElementSize = Interpreter::stackElementSize; |
|
6245 |
int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0); |
|
6246 |
#ifdef ASSERT |
|
6247 |
int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1); |
|
6248 |
assert(offset1 - offset == stackElementSize, "correct arithmetic"); |
|
6249 |
#endif |
|
6250 |
Register scale_reg = noreg; |
|
6251 |
Address::ScaleFactor scale_factor = Address::no_scale; |
|
6252 |
if (arg_slot.is_constant()) { |
|
6253 |
offset += arg_slot.as_constant() * stackElementSize; |
|
6254 |
} else { |
|
6255 |
scale_reg = arg_slot.as_register(); |
|
6256 |
scale_factor = Address::times(stackElementSize); |
|
6257 |
} |
|
6258 |
offset += wordSize; // return PC is on stack |
|
6259 |
return Address(rsp, scale_reg, scale_factor, offset); |
|
6260 |
} |
|
6261 |
||
6262 |
||
6263 |
void MacroAssembler::verify_oop_addr(Address addr, const char* s) { |
|
6264 |
if (!VerifyOops) return; |
|
6265 |
||
6266 |
// Address adjust(addr.base(), addr.index(), addr.scale(), addr.disp() + BytesPerWord); |
|
6267 |
// Pass register number to verify_oop_subroutine |
|
16368
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
6268 |
const char* b = NULL; |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
6269 |
{ |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
6270 |
ResourceMark rm; |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
6271 |
stringStream ss; |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
6272 |
ss.print("verify_oop_addr: %s", s); |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
6273 |
b = code_string(ss.as_string()); |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
6274 |
} |
14626 | 6275 |
#ifdef _LP64 |
6276 |
push(rscratch1); // save r10, trashed by movptr() |
|
6277 |
#endif |
|
6278 |
push(rax); // save rax, |
|
6279 |
// addr may contain rsp so we will have to adjust it based on the push |
|
6280 |
// we just did (and on 64 bit we do two pushes) |
|
6281 |
// NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which |
|
6282 |
// stores rax into addr which is backwards of what was intended. |
|
6283 |
if (addr.uses(rsp)) { |
|
6284 |
lea(rax, addr); |
|
6285 |
pushptr(Address(rax, LP64_ONLY(2 *) BytesPerWord)); |
|
6286 |
} else { |
|
6287 |
pushptr(addr); |
|
6288 |
} |
|
6289 |
||
6290 |
ExternalAddress buffer((address) b); |
|
6291 |
// pass msg argument |
|
6292 |
// avoid using pushptr, as it modifies scratch registers |
|
6293 |
// and our contract is not to modify anything |
|
6294 |
movptr(rax, buffer.addr()); |
|
6295 |
push(rax); |
|
6296 |
||
6297 |
// call indirectly to solve generation ordering problem |
|
6298 |
movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address())); |
|
6299 |
call(rax); |
|
6300 |
// Caller pops the arguments (addr, message) and restores rax, r10. |
|
6301 |
} |
|
6302 |
||
6303 |
void MacroAssembler::verify_tlab() { |
|
6304 |
#ifdef ASSERT |
|
6305 |
if (UseTLAB && VerifyOops) { |
|
6306 |
Label next, ok; |
|
6307 |
Register t1 = rsi; |
|
6308 |
Register thread_reg = NOT_LP64(rbx) LP64_ONLY(r15_thread); |
|
6309 |
||
6310 |
push(t1); |
|
6311 |
NOT_LP64(push(thread_reg)); |
|
6312 |
NOT_LP64(get_thread(thread_reg)); |
|
6313 |
||
6314 |
movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset()))); |
|
6315 |
cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset()))); |
|
6316 |
jcc(Assembler::aboveEqual, next); |
|
6317 |
STOP("assert(top >= start)"); |
|
6318 |
should_not_reach_here(); |
|
6319 |
||
6320 |
bind(next); |
|
6321 |
movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset()))); |
|
6322 |
cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset()))); |
|
6323 |
jcc(Assembler::aboveEqual, ok); |
|
6324 |
STOP("assert(top <= end)"); |
|
6325 |
should_not_reach_here(); |
|
6326 |
||
6327 |
bind(ok); |
|
6328 |
NOT_LP64(pop(thread_reg)); |
|
6329 |
pop(t1); |
|
6330 |
} |
|
6331 |
#endif |
|
6332 |
} |
|
6333 |
||
6334 |
class ControlWord { |
|
6335 |
public: |
|
6336 |
int32_t _value; |
|
6337 |
||
6338 |
int rounding_control() const { return (_value >> 10) & 3 ; } |
|
6339 |
int precision_control() const { return (_value >> 8) & 3 ; } |
|
6340 |
bool precision() const { return ((_value >> 5) & 1) != 0; } |
|
6341 |
bool underflow() const { return ((_value >> 4) & 1) != 0; } |
|
6342 |
bool overflow() const { return ((_value >> 3) & 1) != 0; } |
|
6343 |
bool zero_divide() const { return ((_value >> 2) & 1) != 0; } |
|
6344 |
bool denormalized() const { return ((_value >> 1) & 1) != 0; } |
|
6345 |
bool invalid() const { return ((_value >> 0) & 1) != 0; } |
|
6346 |
||
6347 |
void print() const { |
|
6348 |
// rounding control |
|
6349 |
const char* rc; |
|
6350 |
switch (rounding_control()) { |
|
6351 |
case 0: rc = "round near"; break; |
|
6352 |
case 1: rc = "round down"; break; |
|
6353 |
case 2: rc = "round up "; break; |
|
6354 |
case 3: rc = "chop "; break; |
|
6355 |
}; |
|
6356 |
// precision control |
|
6357 |
const char* pc; |
|
6358 |
switch (precision_control()) { |
|
6359 |
case 0: pc = "24 bits "; break; |
|
6360 |
case 1: pc = "reserved"; break; |
|
6361 |
case 2: pc = "53 bits "; break; |
|
6362 |
case 3: pc = "64 bits "; break; |
|
6363 |
}; |
|
6364 |
// flags |
|
6365 |
char f[9]; |
|
6366 |
f[0] = ' '; |
|
6367 |
f[1] = ' '; |
|
6368 |
f[2] = (precision ()) ? 'P' : 'p'; |
|
6369 |
f[3] = (underflow ()) ? 'U' : 'u'; |
|
6370 |
f[4] = (overflow ()) ? 'O' : 'o'; |
|
6371 |
f[5] = (zero_divide ()) ? 'Z' : 'z'; |
|
6372 |
f[6] = (denormalized()) ? 'D' : 'd'; |
|
6373 |
f[7] = (invalid ()) ? 'I' : 'i'; |
|
6374 |
f[8] = '\x0'; |
|
6375 |
// output |
|
6376 |
printf("%04x masks = %s, %s, %s", _value & 0xFFFF, f, rc, pc); |
|
6377 |
} |
|
6378 |
||
6379 |
}; |
|
6380 |
||
6381 |
class StatusWord { |
|
6382 |
public: |
|
6383 |
int32_t _value; |
|
6384 |
||
6385 |
bool busy() const { return ((_value >> 15) & 1) != 0; } |
|
6386 |
bool C3() const { return ((_value >> 14) & 1) != 0; } |
|
6387 |
bool C2() const { return ((_value >> 10) & 1) != 0; } |
|
6388 |
bool C1() const { return ((_value >> 9) & 1) != 0; } |
|
6389 |
bool C0() const { return ((_value >> 8) & 1) != 0; } |
|
6390 |
int top() const { return (_value >> 11) & 7 ; } |
|
6391 |
bool error_status() const { return ((_value >> 7) & 1) != 0; } |
|
6392 |
bool stack_fault() const { return ((_value >> 6) & 1) != 0; } |
|
6393 |
bool precision() const { return ((_value >> 5) & 1) != 0; } |
|
6394 |
bool underflow() const { return ((_value >> 4) & 1) != 0; } |
|
6395 |
bool overflow() const { return ((_value >> 3) & 1) != 0; } |
|
6396 |
bool zero_divide() const { return ((_value >> 2) & 1) != 0; } |
|
6397 |
bool denormalized() const { return ((_value >> 1) & 1) != 0; } |
|
6398 |
bool invalid() const { return ((_value >> 0) & 1) != 0; } |
|
6399 |
||
6400 |
void print() const { |
|
6401 |
// condition codes |
|
6402 |
char c[5]; |
|
6403 |
c[0] = (C3()) ? '3' : '-'; |
|
6404 |
c[1] = (C2()) ? '2' : '-'; |
|
6405 |
c[2] = (C1()) ? '1' : '-'; |
|
6406 |
c[3] = (C0()) ? '0' : '-'; |
|
6407 |
c[4] = '\x0'; |
|
6408 |
// flags |
|
6409 |
char f[9]; |
|
6410 |
f[0] = (error_status()) ? 'E' : '-'; |
|
6411 |
f[1] = (stack_fault ()) ? 'S' : '-'; |
|
6412 |
f[2] = (precision ()) ? 'P' : '-'; |
|
6413 |
f[3] = (underflow ()) ? 'U' : '-'; |
|
6414 |
f[4] = (overflow ()) ? 'O' : '-'; |
|
6415 |
f[5] = (zero_divide ()) ? 'Z' : '-'; |
|
6416 |
f[6] = (denormalized()) ? 'D' : '-'; |
|
6417 |
f[7] = (invalid ()) ? 'I' : '-'; |
|
6418 |
f[8] = '\x0'; |
|
6419 |
// output |
|
6420 |
printf("%04x flags = %s, cc = %s, top = %d", _value & 0xFFFF, f, c, top()); |
|
6421 |
} |
|
6422 |
||
6423 |
}; |
|
6424 |
||
6425 |
class TagWord { |
|
6426 |
public: |
|
6427 |
int32_t _value; |
|
6428 |
||
6429 |
int tag_at(int i) const { return (_value >> (i*2)) & 3; } |
|
6430 |
||
6431 |
void print() const { |
|
6432 |
printf("%04x", _value & 0xFFFF); |
|
6433 |
} |
|
6434 |
||
6435 |
}; |
|
6436 |
||
6437 |
class FPU_Register { |
|
6438 |
public: |
|
6439 |
int32_t _m0; |
|
6440 |
int32_t _m1; |
|
6441 |
int16_t _ex; |
|
6442 |
||
6443 |
bool is_indefinite() const { |
|
6444 |
return _ex == -1 && _m1 == (int32_t)0xC0000000 && _m0 == 0; |
|
6445 |
} |
|
6446 |
||
6447 |
void print() const { |
|
6448 |
char sign = (_ex < 0) ? '-' : '+'; |
|
6449 |
const char* kind = (_ex == 0x7FFF || _ex == (int16_t)-1) ? "NaN" : " "; |
|
6450 |
printf("%c%04hx.%08x%08x %s", sign, _ex, _m1, _m0, kind); |
|
6451 |
}; |
|
6452 |
||
6453 |
}; |
|
6454 |
||
6455 |
class FPU_State { |
|
6456 |
public: |
|
6457 |
enum { |
|
6458 |
register_size = 10, |
|
6459 |
number_of_registers = 8, |
|
6460 |
register_mask = 7 |
|
6461 |
}; |
|
6462 |
||
6463 |
ControlWord _control_word; |
|
6464 |
StatusWord _status_word; |
|
6465 |
TagWord _tag_word; |
|
6466 |
int32_t _error_offset; |
|
6467 |
int32_t _error_selector; |
|
6468 |
int32_t _data_offset; |
|
6469 |
int32_t _data_selector; |
|
6470 |
int8_t _register[register_size * number_of_registers]; |
|
6471 |
||
6472 |
int tag_for_st(int i) const { return _tag_word.tag_at((_status_word.top() + i) & register_mask); } |
|
6473 |
FPU_Register* st(int i) const { return (FPU_Register*)&_register[register_size * i]; } |
|
6474 |
||
6475 |
const char* tag_as_string(int tag) const { |
|
6476 |
switch (tag) { |
|
6477 |
case 0: return "valid"; |
|
6478 |
case 1: return "zero"; |
|
6479 |
case 2: return "special"; |
|
6480 |
case 3: return "empty"; |
|
6481 |
} |
|
6482 |
ShouldNotReachHere(); |
|
6483 |
return NULL; |
|
6484 |
} |
|
6485 |
||
6486 |
void print() const { |
|
6487 |
// print computation registers |
|
6488 |
{ int t = _status_word.top(); |
|
6489 |
for (int i = 0; i < number_of_registers; i++) { |
|
6490 |
int j = (i - t) & register_mask; |
|
6491 |
printf("%c r%d = ST%d = ", (j == 0 ? '*' : ' '), i, j); |
|
6492 |
st(j)->print(); |
|
6493 |
printf(" %s\n", tag_as_string(_tag_word.tag_at(i))); |
|
6494 |
} |
|
6495 |
} |
|
6496 |
printf("\n"); |
|
6497 |
// print control registers |
|
6498 |
printf("ctrl = "); _control_word.print(); printf("\n"); |
|
6499 |
printf("stat = "); _status_word .print(); printf("\n"); |
|
6500 |
printf("tags = "); _tag_word .print(); printf("\n"); |
|
6501 |
} |
|
6502 |
||
6503 |
}; |
|
6504 |
||
6505 |
class Flag_Register { |
|
6506 |
public: |
|
6507 |
int32_t _value; |
|
6508 |
||
6509 |
bool overflow() const { return ((_value >> 11) & 1) != 0; } |
|
6510 |
bool direction() const { return ((_value >> 10) & 1) != 0; } |
|
6511 |
bool sign() const { return ((_value >> 7) & 1) != 0; } |
|
6512 |
bool zero() const { return ((_value >> 6) & 1) != 0; } |
|
6513 |
bool auxiliary_carry() const { return ((_value >> 4) & 1) != 0; } |
|
6514 |
bool parity() const { return ((_value >> 2) & 1) != 0; } |
|
6515 |
bool carry() const { return ((_value >> 0) & 1) != 0; } |
|
6516 |
||
6517 |
void print() const { |
|
6518 |
// flags |
|
6519 |
char f[8]; |
|
6520 |
f[0] = (overflow ()) ? 'O' : '-'; |
|
6521 |
f[1] = (direction ()) ? 'D' : '-'; |
|
6522 |
f[2] = (sign ()) ? 'S' : '-'; |
|
6523 |
f[3] = (zero ()) ? 'Z' : '-'; |
|
6524 |
f[4] = (auxiliary_carry()) ? 'A' : '-'; |
|
6525 |
f[5] = (parity ()) ? 'P' : '-'; |
|
6526 |
f[6] = (carry ()) ? 'C' : '-'; |
|
6527 |
f[7] = '\x0'; |
|
6528 |
// output |
|
6529 |
printf("%08x flags = %s", _value, f); |
|
6530 |
} |
|
6531 |
||
6532 |
}; |
|
6533 |
||
6534 |
class IU_Register { |
|
6535 |
public: |
|
6536 |
int32_t _value; |
|
6537 |
||
6538 |
void print() const { |
|
6539 |
printf("%08x %11d", _value, _value); |
|
6540 |
} |
|
6541 |
||
6542 |
}; |
|
6543 |
||
6544 |
class IU_State { |
|
6545 |
public: |
|
6546 |
Flag_Register _eflags; |
|
6547 |
IU_Register _rdi; |
|
6548 |
IU_Register _rsi; |
|
6549 |
IU_Register _rbp; |
|
6550 |
IU_Register _rsp; |
|
6551 |
IU_Register _rbx; |
|
6552 |
IU_Register _rdx; |
|
6553 |
IU_Register _rcx; |
|
6554 |
IU_Register _rax; |
|
6555 |
||
6556 |
void print() const { |
|
6557 |
// computation registers |
|
6558 |
printf("rax, = "); _rax.print(); printf("\n"); |
|
6559 |
printf("rbx, = "); _rbx.print(); printf("\n"); |
|
6560 |
printf("rcx = "); _rcx.print(); printf("\n"); |
|
6561 |
printf("rdx = "); _rdx.print(); printf("\n"); |
|
6562 |
printf("rdi = "); _rdi.print(); printf("\n"); |
|
6563 |
printf("rsi = "); _rsi.print(); printf("\n"); |
|
6564 |
printf("rbp, = "); _rbp.print(); printf("\n"); |
|
6565 |
printf("rsp = "); _rsp.print(); printf("\n"); |
|
6566 |
printf("\n"); |
|
6567 |
// control registers |
|
6568 |
printf("flgs = "); _eflags.print(); printf("\n"); |
|
6569 |
} |
|
6570 |
}; |
|
6571 |
||
6572 |
||
6573 |
class CPU_State { |
|
6574 |
public: |
|
6575 |
FPU_State _fpu_state; |
|
6576 |
IU_State _iu_state; |
|
6577 |
||
6578 |
void print() const { |
|
6579 |
printf("--------------------------------------------------\n"); |
|
6580 |
_iu_state .print(); |
|
6581 |
printf("\n"); |
|
6582 |
_fpu_state.print(); |
|
6583 |
printf("--------------------------------------------------\n"); |
|
6584 |
} |
|
6585 |
||
6586 |
}; |
|
6587 |
||
6588 |
||
6589 |
static void _print_CPU_state(CPU_State* state) { |
|
6590 |
state->print(); |
|
6591 |
}; |
|
6592 |
||
6593 |
||
6594 |
void MacroAssembler::print_CPU_state() { |
|
6595 |
push_CPU_state(); |
|
6596 |
push(rsp); // pass CPU state |
|
6597 |
call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state))); |
|
6598 |
addptr(rsp, wordSize); // discard argument |
|
6599 |
pop_CPU_state(); |
|
6600 |
} |
|
6601 |
||
6602 |
||
6603 |
static bool _verify_FPU(int stack_depth, char* s, CPU_State* state) { |
|
6604 |
static int counter = 0; |
|
6605 |
FPU_State* fs = &state->_fpu_state; |
|
6606 |
counter++; |
|
6607 |
// For leaf calls, only verify that the top few elements remain empty. |
|
6608 |
// We only need 1 empty at the top for C2 code. |
|
6609 |
if( stack_depth < 0 ) { |
|
6610 |
if( fs->tag_for_st(7) != 3 ) { |
|
6611 |
printf("FPR7 not empty\n"); |
|
6612 |
state->print(); |
|
6613 |
assert(false, "error"); |
|
6614 |
return false; |
|
6615 |
} |
|
6616 |
return true; // All other stack states do not matter |
|
6617 |
} |
|
6618 |
||
6619 |
assert((fs->_control_word._value & 0xffff) == StubRoutines::_fpu_cntrl_wrd_std, |
|
6620 |
"bad FPU control word"); |
|
6621 |
||
6622 |
// compute stack depth |
|
6623 |
int i = 0; |
|
6624 |
while (i < FPU_State::number_of_registers && fs->tag_for_st(i) < 3) i++; |
|
6625 |
int d = i; |
|
6626 |
while (i < FPU_State::number_of_registers && fs->tag_for_st(i) == 3) i++; |
|
6627 |
// verify findings |
|
6628 |
if (i != FPU_State::number_of_registers) { |
|
6629 |
// stack not contiguous |
|
6630 |
printf("%s: stack not contiguous at ST%d\n", s, i); |
|
6631 |
state->print(); |
|
6632 |
assert(false, "error"); |
|
6633 |
return false; |
|
6634 |
} |
|
6635 |
// check if computed stack depth corresponds to expected stack depth |
|
6636 |
if (stack_depth < 0) { |
|
6637 |
// expected stack depth is -stack_depth or less |
|
6638 |
if (d > -stack_depth) { |
|
6639 |
// too many elements on the stack |
|
6640 |
printf("%s: <= %d stack elements expected but found %d\n", s, -stack_depth, d); |
|
6641 |
state->print(); |
|
6642 |
assert(false, "error"); |
|
6643 |
return false; |
|
6644 |
} |
|
6645 |
} else { |
|
6646 |
// expected stack depth is stack_depth |
|
6647 |
if (d != stack_depth) { |
|
6648 |
// wrong stack depth |
|
6649 |
printf("%s: %d stack elements expected but found %d\n", s, stack_depth, d); |
|
6650 |
state->print(); |
|
6651 |
assert(false, "error"); |
|
6652 |
return false; |
|
6653 |
} |
|
6654 |
} |
|
6655 |
// everything is cool |
|
6656 |
return true; |
|
6657 |
} |
|
6658 |
||
6659 |
||
6660 |
void MacroAssembler::verify_FPU(int stack_depth, const char* s) { |
|
6661 |
if (!VerifyFPU) return; |
|
6662 |
push_CPU_state(); |
|
6663 |
push(rsp); // pass CPU state |
|
6664 |
ExternalAddress msg((address) s); |
|
6665 |
// pass message string s |
|
6666 |
pushptr(msg.addr()); |
|
6667 |
push(stack_depth); // pass stack depth |
|
6668 |
call(RuntimeAddress(CAST_FROM_FN_PTR(address, _verify_FPU))); |
|
6669 |
addptr(rsp, 3 * wordSize); // discard arguments |
|
6670 |
// check for error |
|
6671 |
{ Label L; |
|
6672 |
testl(rax, rax); |
|
6673 |
jcc(Assembler::notZero, L); |
|
6674 |
int3(); // break if error condition |
|
6675 |
bind(L); |
|
6676 |
} |
|
6677 |
pop_CPU_state(); |
|
6678 |
} |
|
6679 |
||
16624
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
6680 |
void MacroAssembler::restore_cpu_control_state_after_jni() { |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
6681 |
// Either restore the MXCSR register after returning from the JNI Call |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
6682 |
// or verify that it wasn't changed (with -Xcheck:jni flag). |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
6683 |
if (VM_Version::supports_sse()) { |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
6684 |
if (RestoreMXCSROnJNICalls) { |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
6685 |
ldmxcsr(ExternalAddress(StubRoutines::addr_mxcsr_std())); |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
6686 |
} else if (CheckJNICalls) { |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
6687 |
call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry())); |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
6688 |
} |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
6689 |
} |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
6690 |
if (VM_Version::supports_avx()) { |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
6691 |
// Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty. |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
6692 |
vzeroupper(); |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
6693 |
} |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
6694 |
|
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
6695 |
#ifndef _LP64 |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
6696 |
// Either restore the x87 floating pointer control word after returning |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
6697 |
// from the JNI call or verify that it wasn't changed. |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
6698 |
if (CheckJNICalls) { |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
6699 |
call(RuntimeAddress(StubRoutines::x86::verify_fpu_cntrl_wrd_entry())); |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
6700 |
} |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
6701 |
#endif // _LP64 |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
6702 |
} |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
6703 |
|
38074
8475fdc6dcc3
8154580: Save mirror in interpreter frame to enable cleanups of CLDClosure
coleenp
parents:
38049
diff
changeset
|
6704 |
void MacroAssembler::load_mirror(Register mirror, Register method) { |
8475fdc6dcc3
8154580: Save mirror in interpreter frame to enable cleanups of CLDClosure
coleenp
parents:
38049
diff
changeset
|
6705 |
// get mirror |
8475fdc6dcc3
8154580: Save mirror in interpreter frame to enable cleanups of CLDClosure
coleenp
parents:
38049
diff
changeset
|
6706 |
const int mirror_offset = in_bytes(Klass::java_mirror_offset()); |
8475fdc6dcc3
8154580: Save mirror in interpreter frame to enable cleanups of CLDClosure
coleenp
parents:
38049
diff
changeset
|
6707 |
movptr(mirror, Address(method, Method::const_offset())); |
8475fdc6dcc3
8154580: Save mirror in interpreter frame to enable cleanups of CLDClosure
coleenp
parents:
38049
diff
changeset
|
6708 |
movptr(mirror, Address(mirror, ConstMethod::constants_offset())); |
8475fdc6dcc3
8154580: Save mirror in interpreter frame to enable cleanups of CLDClosure
coleenp
parents:
38049
diff
changeset
|
6709 |
movptr(mirror, Address(mirror, ConstantPool::pool_holder_offset_in_bytes())); |
8475fdc6dcc3
8154580: Save mirror in interpreter frame to enable cleanups of CLDClosure
coleenp
parents:
38049
diff
changeset
|
6710 |
movptr(mirror, Address(mirror, mirror_offset)); |
8475fdc6dcc3
8154580: Save mirror in interpreter frame to enable cleanups of CLDClosure
coleenp
parents:
38049
diff
changeset
|
6711 |
} |
16624
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
6712 |
|
14626 | 6713 |
void MacroAssembler::load_klass(Register dst, Register src) { |
6714 |
#ifdef _LP64 |
|
19979
ebe1dbb6e1aa
8015107: NPG: Use consistent naming for metaspace concepts
ehelin
parents:
19319
diff
changeset
|
6715 |
if (UseCompressedClassPointers) { |
14626 | 6716 |
movl(dst, Address(src, oopDesc::klass_offset_in_bytes())); |
6717 |
decode_klass_not_null(dst); |
|
6718 |
} else |
|
6719 |
#endif |
|
6720 |
movptr(dst, Address(src, oopDesc::klass_offset_in_bytes())); |
|
6721 |
} |
|
6722 |
||
6723 |
void MacroAssembler::load_prototype_header(Register dst, Register src) { |
|
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
6724 |
load_klass(dst, src); |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
6725 |
movptr(dst, Address(dst, Klass::prototype_header_offset())); |
14626 | 6726 |
} |
6727 |
||
6728 |
void MacroAssembler::store_klass(Register dst, Register src) { |
|
6729 |
#ifdef _LP64 |
|
19979
ebe1dbb6e1aa
8015107: NPG: Use consistent naming for metaspace concepts
ehelin
parents:
19319
diff
changeset
|
6730 |
if (UseCompressedClassPointers) { |
14626 | 6731 |
encode_klass_not_null(src); |
6732 |
movl(Address(dst, oopDesc::klass_offset_in_bytes()), src); |
|
6733 |
} else |
|
6734 |
#endif |
|
6735 |
movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src); |
|
6736 |
} |
|
6737 |
||
6738 |
void MacroAssembler::load_heap_oop(Register dst, Address src) { |
|
6739 |
#ifdef _LP64 |
|
6740 |
// FIXME: Must change all places where we try to load the klass. |
|
6741 |
if (UseCompressedOops) { |
|
6742 |
movl(dst, src); |
|
6743 |
decode_heap_oop(dst); |
|
6744 |
} else |
|
6745 |
#endif |
|
6746 |
movptr(dst, src); |
|
6747 |
} |
|
6748 |
||
6749 |
// Doesn't do verfication, generates fixed size code |
|
6750 |
void MacroAssembler::load_heap_oop_not_null(Register dst, Address src) { |
|
6751 |
#ifdef _LP64 |
|
6752 |
if (UseCompressedOops) { |
|
6753 |
movl(dst, src); |
|
6754 |
decode_heap_oop_not_null(dst); |
|
6755 |
} else |
|
6756 |
#endif |
|
6757 |
movptr(dst, src); |
|
6758 |
} |
|
6759 |
||
6760 |
void MacroAssembler::store_heap_oop(Address dst, Register src) { |
|
6761 |
#ifdef _LP64 |
|
6762 |
if (UseCompressedOops) { |
|
6763 |
assert(!dst.uses(src), "not enough registers"); |
|
6764 |
encode_heap_oop(src); |
|
6765 |
movl(dst, src); |
|
6766 |
} else |
|
6767 |
#endif |
|
6768 |
movptr(dst, src); |
|
6769 |
} |
|
6770 |
||
6771 |
void MacroAssembler::cmp_heap_oop(Register src1, Address src2, Register tmp) { |
|
6772 |
assert_different_registers(src1, tmp); |
|
6773 |
#ifdef _LP64 |
|
6774 |
if (UseCompressedOops) { |
|
6775 |
bool did_push = false; |
|
6776 |
if (tmp == noreg) { |
|
6777 |
tmp = rax; |
|
6778 |
push(tmp); |
|
6779 |
did_push = true; |
|
6780 |
assert(!src2.uses(rsp), "can't push"); |
|
6781 |
} |
|
6782 |
load_heap_oop(tmp, src2); |
|
6783 |
cmpptr(src1, tmp); |
|
6784 |
if (did_push) pop(tmp); |
|
6785 |
} else |
|
6786 |
#endif |
|
6787 |
cmpptr(src1, src2); |
|
6788 |
} |
|
6789 |
||
6790 |
// Used for storing NULLs. |
|
6791 |
void MacroAssembler::store_heap_oop_null(Address dst) { |
|
6792 |
#ifdef _LP64 |
|
6793 |
if (UseCompressedOops) { |
|
6794 |
movl(dst, (int32_t)NULL_WORD); |
|
6795 |
} else { |
|
6796 |
movslq(dst, (int32_t)NULL_WORD); |
|
6797 |
} |
|
6798 |
#else |
|
6799 |
movl(dst, (int32_t)NULL_WORD); |
|
6800 |
#endif |
|
6801 |
} |
|
6802 |
||
6803 |
#ifdef _LP64 |
|
6804 |
void MacroAssembler::store_klass_gap(Register dst, Register src) { |
|
19979
ebe1dbb6e1aa
8015107: NPG: Use consistent naming for metaspace concepts
ehelin
parents:
19319
diff
changeset
|
6805 |
if (UseCompressedClassPointers) { |
14626 | 6806 |
// Store to klass gap in destination |
6807 |
movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src); |
|
6808 |
} |
|
6809 |
} |
|
6810 |
||
6811 |
#ifdef ASSERT |
|
6812 |
void MacroAssembler::verify_heapbase(const char* msg) { |
|
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
6813 |
assert (UseCompressedOops, "should be compressed"); |
14626 | 6814 |
assert (Universe::heap() != NULL, "java heap should be initialized"); |
6815 |
if (CheckCompressedOops) { |
|
6816 |
Label ok; |
|
6817 |
push(rscratch1); // cmpptr trashes rscratch1 |
|
6818 |
cmpptr(r12_heapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr())); |
|
6819 |
jcc(Assembler::equal, ok); |
|
6820 |
STOP(msg); |
|
6821 |
bind(ok); |
|
6822 |
pop(rscratch1); |
|
6823 |
} |
|
6824 |
} |
|
6825 |
#endif |
|
6826 |
||
6827 |
// Algorithm must match oop.inline.hpp encode_heap_oop. |
|
6828 |
void MacroAssembler::encode_heap_oop(Register r) { |
|
6829 |
#ifdef ASSERT |
|
6830 |
verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?"); |
|
6831 |
#endif |
|
6832 |
verify_oop(r, "broken oop in encode_heap_oop"); |
|
6833 |
if (Universe::narrow_oop_base() == NULL) { |
|
6834 |
if (Universe::narrow_oop_shift() != 0) { |
|
6835 |
assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong"); |
|
6836 |
shrq(r, LogMinObjAlignmentInBytes); |
|
6837 |
} |
|
6838 |
return; |
|
6839 |
} |
|
6840 |
testq(r, r); |
|
6841 |
cmovq(Assembler::equal, r, r12_heapbase); |
|
6842 |
subq(r, r12_heapbase); |
|
6843 |
shrq(r, LogMinObjAlignmentInBytes); |
|
6844 |
} |
|
6845 |
||
6846 |
void MacroAssembler::encode_heap_oop_not_null(Register r) { |
|
6847 |
#ifdef ASSERT |
|
6848 |
verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?"); |
|
6849 |
if (CheckCompressedOops) { |
|
6850 |
Label ok; |
|
6851 |
testq(r, r); |
|
6852 |
jcc(Assembler::notEqual, ok); |
|
6853 |
STOP("null oop passed to encode_heap_oop_not_null"); |
|
6854 |
bind(ok); |
|
6855 |
} |
|
6856 |
#endif |
|
6857 |
verify_oop(r, "broken oop in encode_heap_oop_not_null"); |
|
6858 |
if (Universe::narrow_oop_base() != NULL) { |
|
6859 |
subq(r, r12_heapbase); |
|
6860 |
} |
|
6861 |
if (Universe::narrow_oop_shift() != 0) { |
|
6862 |
assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong"); |
|
6863 |
shrq(r, LogMinObjAlignmentInBytes); |
|
6864 |
} |
|
6865 |
} |
|
6866 |
||
6867 |
void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) { |
|
6868 |
#ifdef ASSERT |
|
6869 |
verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?"); |
|
6870 |
if (CheckCompressedOops) { |
|
6871 |
Label ok; |
|
6872 |
testq(src, src); |
|
6873 |
jcc(Assembler::notEqual, ok); |
|
6874 |
STOP("null oop passed to encode_heap_oop_not_null2"); |
|
6875 |
bind(ok); |
|
6876 |
} |
|
6877 |
#endif |
|
6878 |
verify_oop(src, "broken oop in encode_heap_oop_not_null2"); |
|
6879 |
if (dst != src) { |
|
6880 |
movq(dst, src); |
|
6881 |
} |
|
6882 |
if (Universe::narrow_oop_base() != NULL) { |
|
6883 |
subq(dst, r12_heapbase); |
|
6884 |
} |
|
6885 |
if (Universe::narrow_oop_shift() != 0) { |
|
6886 |
assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong"); |
|
6887 |
shrq(dst, LogMinObjAlignmentInBytes); |
|
6888 |
} |
|
6889 |
} |
|
6890 |
||
6891 |
void MacroAssembler::decode_heap_oop(Register r) { |
|
6892 |
#ifdef ASSERT |
|
6893 |
verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?"); |
|
6894 |
#endif |
|
6895 |
if (Universe::narrow_oop_base() == NULL) { |
|
6896 |
if (Universe::narrow_oop_shift() != 0) { |
|
6897 |
assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong"); |
|
6898 |
shlq(r, LogMinObjAlignmentInBytes); |
|
6899 |
} |
|
6900 |
} else { |
|
6901 |
Label done; |
|
6902 |
shlq(r, LogMinObjAlignmentInBytes); |
|
6903 |
jccb(Assembler::equal, done); |
|
6904 |
addq(r, r12_heapbase); |
|
6905 |
bind(done); |
|
6906 |
} |
|
6907 |
verify_oop(r, "broken oop in decode_heap_oop"); |
|
6908 |
} |
|
6909 |
||
6910 |
void MacroAssembler::decode_heap_oop_not_null(Register r) { |
|
6911 |
// Note: it will change flags |
|
6912 |
assert (UseCompressedOops, "should only be used for compressed headers"); |
|
6913 |
assert (Universe::heap() != NULL, "java heap should be initialized"); |
|
6914 |
// Cannot assert, unverified entry point counts instructions (see .ad file) |
|
6915 |
// vtableStubs also counts instructions in pd_code_size_limit. |
|
6916 |
// Also do not verify_oop as this is called by verify_oop. |
|
6917 |
if (Universe::narrow_oop_shift() != 0) { |
|
6918 |
assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong"); |
|
6919 |
shlq(r, LogMinObjAlignmentInBytes); |
|
6920 |
if (Universe::narrow_oop_base() != NULL) { |
|
6921 |
addq(r, r12_heapbase); |
|
6922 |
} |
|
6923 |
} else { |
|
6924 |
assert (Universe::narrow_oop_base() == NULL, "sanity"); |
|
6925 |
} |
|
6926 |
} |
|
6927 |
||
6928 |
void MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) { |
|
6929 |
// Note: it will change flags |
|
6930 |
assert (UseCompressedOops, "should only be used for compressed headers"); |
|
6931 |
assert (Universe::heap() != NULL, "java heap should be initialized"); |
|
6932 |
// Cannot assert, unverified entry point counts instructions (see .ad file) |
|
6933 |
// vtableStubs also counts instructions in pd_code_size_limit. |
|
6934 |
// Also do not verify_oop as this is called by verify_oop. |
|
6935 |
if (Universe::narrow_oop_shift() != 0) { |
|
6936 |
assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong"); |
|
6937 |
if (LogMinObjAlignmentInBytes == Address::times_8) { |
|
6938 |
leaq(dst, Address(r12_heapbase, src, Address::times_8, 0)); |
|
6939 |
} else { |
|
6940 |
if (dst != src) { |
|
6941 |
movq(dst, src); |
|
6942 |
} |
|
6943 |
shlq(dst, LogMinObjAlignmentInBytes); |
|
6944 |
if (Universe::narrow_oop_base() != NULL) { |
|
6945 |
addq(dst, r12_heapbase); |
|
6946 |
} |
|
6947 |
} |
|
6948 |
} else { |
|
6949 |
assert (Universe::narrow_oop_base() == NULL, "sanity"); |
|
6950 |
if (dst != src) { |
|
6951 |
movq(dst, src); |
|
6952 |
} |
|
6953 |
} |
|
6954 |
} |
|
6955 |
||
6956 |
void MacroAssembler::encode_klass_not_null(Register r) { |
|
21188
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
6957 |
if (Universe::narrow_klass_base() != NULL) { |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
6958 |
// Use r12 as a scratch register in which to temporarily load the narrow_klass_base. |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
6959 |
assert(r != r12_heapbase, "Encoding a klass in r12"); |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
6960 |
mov64(r12_heapbase, (int64_t)Universe::narrow_klass_base()); |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
6961 |
subq(r, r12_heapbase); |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
6962 |
} |
14626 | 6963 |
if (Universe::narrow_klass_shift() != 0) { |
6964 |
assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong"); |
|
6965 |
shrq(r, LogKlassAlignmentInBytes); |
|
6966 |
} |
|
21188
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
6967 |
if (Universe::narrow_klass_base() != NULL) { |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
6968 |
reinit_heapbase(); |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
6969 |
} |
14626 | 6970 |
} |
6971 |
||
6972 |
void MacroAssembler::encode_klass_not_null(Register dst, Register src) { |
|
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
6973 |
if (dst == src) { |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
6974 |
encode_klass_not_null(src); |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
6975 |
} else { |
21188
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
6976 |
if (Universe::narrow_klass_base() != NULL) { |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
6977 |
mov64(dst, (int64_t)Universe::narrow_klass_base()); |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
6978 |
negq(dst); |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
6979 |
addq(dst, src); |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
6980 |
} else { |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
6981 |
movptr(dst, src); |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
6982 |
} |
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
6983 |
if (Universe::narrow_klass_shift() != 0) { |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
6984 |
assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong"); |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
6985 |
shrq(dst, LogKlassAlignmentInBytes); |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
6986 |
} |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
6987 |
} |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
6988 |
} |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
6989 |
|
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
6990 |
// Function instr_size_for_decode_klass_not_null() counts the instructions |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
6991 |
// generated by decode_klass_not_null(register r) and reinit_heapbase(), |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
6992 |
// when (Universe::heap() != NULL). Hence, if the instructions they |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
6993 |
// generate change, then this method needs to be updated. |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
6994 |
int MacroAssembler::instr_size_for_decode_klass_not_null() { |
19979
ebe1dbb6e1aa
8015107: NPG: Use consistent naming for metaspace concepts
ehelin
parents:
19319
diff
changeset
|
6995 |
assert (UseCompressedClassPointers, "only for compressed klass ptrs"); |
21188
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
6996 |
if (Universe::narrow_klass_base() != NULL) { |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
6997 |
// mov64 + addq + shlq? + mov64 (for reinit_heapbase()). |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
6998 |
return (Universe::narrow_klass_shift() == 0 ? 20 : 24); |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
6999 |
} else { |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
7000 |
// longest load decode klass function, mov64, leaq |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
7001 |
return 16; |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
7002 |
} |
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7003 |
} |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7004 |
|
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7005 |
// !!! If the instructions that get generated here change then function |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7006 |
// instr_size_for_decode_klass_not_null() needs to get updated. |
14626 | 7007 |
void MacroAssembler::decode_klass_not_null(Register r) { |
7008 |
// Note: it will change flags |
|
19979
ebe1dbb6e1aa
8015107: NPG: Use consistent naming for metaspace concepts
ehelin
parents:
19319
diff
changeset
|
7009 |
assert (UseCompressedClassPointers, "should only be used for compressed headers"); |
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7010 |
assert(r != r12_heapbase, "Decoding a klass in r12"); |
14626 | 7011 |
// Cannot assert, unverified entry point counts instructions (see .ad file) |
7012 |
// vtableStubs also counts instructions in pd_code_size_limit. |
|
7013 |
// Also do not verify_oop as this is called by verify_oop. |
|
7014 |
if (Universe::narrow_klass_shift() != 0) { |
|
7015 |
assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong"); |
|
7016 |
shlq(r, LogKlassAlignmentInBytes); |
|
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7017 |
} |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7018 |
// Use r12 as a scratch register in which to temporarily load the narrow_klass_base. |
21188
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
7019 |
if (Universe::narrow_klass_base() != NULL) { |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
7020 |
mov64(r12_heapbase, (int64_t)Universe::narrow_klass_base()); |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
7021 |
addq(r, r12_heapbase); |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
7022 |
reinit_heapbase(); |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
7023 |
} |
14626 | 7024 |
} |
7025 |
||
7026 |
void MacroAssembler::decode_klass_not_null(Register dst, Register src) { |
|
7027 |
// Note: it will change flags |
|
19979
ebe1dbb6e1aa
8015107: NPG: Use consistent naming for metaspace concepts
ehelin
parents:
19319
diff
changeset
|
7028 |
assert (UseCompressedClassPointers, "should only be used for compressed headers"); |
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7029 |
if (dst == src) { |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7030 |
decode_klass_not_null(dst); |
14626 | 7031 |
} else { |
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7032 |
// Cannot assert, unverified entry point counts instructions (see .ad file) |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7033 |
// vtableStubs also counts instructions in pd_code_size_limit. |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7034 |
// Also do not verify_oop as this is called by verify_oop. |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7035 |
mov64(dst, (int64_t)Universe::narrow_klass_base()); |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7036 |
if (Universe::narrow_klass_shift() != 0) { |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7037 |
assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong"); |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7038 |
assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?"); |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7039 |
leaq(dst, Address(dst, src, Address::times_8, 0)); |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7040 |
} else { |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7041 |
addq(dst, src); |
14626 | 7042 |
} |
7043 |
} |
|
7044 |
} |
|
7045 |
||
7046 |
void MacroAssembler::set_narrow_oop(Register dst, jobject obj) { |
|
7047 |
assert (UseCompressedOops, "should only be used for compressed headers"); |
|
7048 |
assert (Universe::heap() != NULL, "java heap should be initialized"); |
|
7049 |
assert (oop_recorder() != NULL, "this assembler needs an OopRecorder"); |
|
7050 |
int oop_index = oop_recorder()->find_index(obj); |
|
7051 |
RelocationHolder rspec = oop_Relocation::spec(oop_index); |
|
7052 |
mov_narrow_oop(dst, oop_index, rspec); |
|
7053 |
} |
|
7054 |
||
7055 |
void MacroAssembler::set_narrow_oop(Address dst, jobject obj) { |
|
7056 |
assert (UseCompressedOops, "should only be used for compressed headers"); |
|
7057 |
assert (Universe::heap() != NULL, "java heap should be initialized"); |
|
7058 |
assert (oop_recorder() != NULL, "this assembler needs an OopRecorder"); |
|
7059 |
int oop_index = oop_recorder()->find_index(obj); |
|
7060 |
RelocationHolder rspec = oop_Relocation::spec(oop_index); |
|
7061 |
mov_narrow_oop(dst, oop_index, rspec); |
|
7062 |
} |
|
7063 |
||
7064 |
void MacroAssembler::set_narrow_klass(Register dst, Klass* k) { |
|
19979
ebe1dbb6e1aa
8015107: NPG: Use consistent naming for metaspace concepts
ehelin
parents:
19319
diff
changeset
|
7065 |
assert (UseCompressedClassPointers, "should only be used for compressed headers"); |
14626 | 7066 |
assert (oop_recorder() != NULL, "this assembler needs an OopRecorder"); |
7067 |
int klass_index = oop_recorder()->find_index(k); |
|
7068 |
RelocationHolder rspec = metadata_Relocation::spec(klass_index); |
|
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7069 |
mov_narrow_oop(dst, Klass::encode_klass(k), rspec); |
14626 | 7070 |
} |
7071 |
||
7072 |
void MacroAssembler::set_narrow_klass(Address dst, Klass* k) { |
|
19979
ebe1dbb6e1aa
8015107: NPG: Use consistent naming for metaspace concepts
ehelin
parents:
19319
diff
changeset
|
7073 |
assert (UseCompressedClassPointers, "should only be used for compressed headers"); |
14626 | 7074 |
assert (oop_recorder() != NULL, "this assembler needs an OopRecorder"); |
7075 |
int klass_index = oop_recorder()->find_index(k); |
|
7076 |
RelocationHolder rspec = metadata_Relocation::spec(klass_index); |
|
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7077 |
mov_narrow_oop(dst, Klass::encode_klass(k), rspec); |
14626 | 7078 |
} |
7079 |
||
7080 |
void MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) { |
|
7081 |
assert (UseCompressedOops, "should only be used for compressed headers"); |
|
7082 |
assert (Universe::heap() != NULL, "java heap should be initialized"); |
|
7083 |
assert (oop_recorder() != NULL, "this assembler needs an OopRecorder"); |
|
7084 |
int oop_index = oop_recorder()->find_index(obj); |
|
7085 |
RelocationHolder rspec = oop_Relocation::spec(oop_index); |
|
7086 |
Assembler::cmp_narrow_oop(dst, oop_index, rspec); |
|
7087 |
} |
|
7088 |
||
7089 |
void MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) { |
|
7090 |
assert (UseCompressedOops, "should only be used for compressed headers"); |
|
7091 |
assert (Universe::heap() != NULL, "java heap should be initialized"); |
|
7092 |
assert (oop_recorder() != NULL, "this assembler needs an OopRecorder"); |
|
7093 |
int oop_index = oop_recorder()->find_index(obj); |
|
7094 |
RelocationHolder rspec = oop_Relocation::spec(oop_index); |
|
7095 |
Assembler::cmp_narrow_oop(dst, oop_index, rspec); |
|
7096 |
} |
|
7097 |
||
7098 |
void MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) { |
|
19979
ebe1dbb6e1aa
8015107: NPG: Use consistent naming for metaspace concepts
ehelin
parents:
19319
diff
changeset
|
7099 |
assert (UseCompressedClassPointers, "should only be used for compressed headers"); |
14626 | 7100 |
assert (oop_recorder() != NULL, "this assembler needs an OopRecorder"); |
7101 |
int klass_index = oop_recorder()->find_index(k); |
|
7102 |
RelocationHolder rspec = metadata_Relocation::spec(klass_index); |
|
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7103 |
Assembler::cmp_narrow_oop(dst, Klass::encode_klass(k), rspec); |
14626 | 7104 |
} |
7105 |
||
7106 |
void MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) { |
|
19979
ebe1dbb6e1aa
8015107: NPG: Use consistent naming for metaspace concepts
ehelin
parents:
19319
diff
changeset
|
7107 |
assert (UseCompressedClassPointers, "should only be used for compressed headers"); |
14626 | 7108 |
assert (oop_recorder() != NULL, "this assembler needs an OopRecorder"); |
7109 |
int klass_index = oop_recorder()->find_index(k); |
|
7110 |
RelocationHolder rspec = metadata_Relocation::spec(klass_index); |
|
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7111 |
Assembler::cmp_narrow_oop(dst, Klass::encode_klass(k), rspec); |
14626 | 7112 |
} |
7113 |
||
7114 |
void MacroAssembler::reinit_heapbase() { |
|
19979
ebe1dbb6e1aa
8015107: NPG: Use consistent naming for metaspace concepts
ehelin
parents:
19319
diff
changeset
|
7115 |
if (UseCompressedOops || UseCompressedClassPointers) { |
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7116 |
if (Universe::heap() != NULL) { |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7117 |
if (Universe::narrow_oop_base() == NULL) { |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7118 |
MacroAssembler::xorptr(r12_heapbase, r12_heapbase); |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7119 |
} else { |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7120 |
mov64(r12_heapbase, (int64_t)Universe::narrow_ptrs_base()); |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7121 |
} |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7122 |
} else { |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7123 |
movptr(r12_heapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr())); |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7124 |
} |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7125 |
} |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7126 |
} |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
7127 |
|
14626 | 7128 |
#endif // _LP64 |
7129 |
||
7130 |
||
7131 |
// C2 compiled method's prolog code. |
|
24018
77b156916bab
8032410: compiler/uncommontrap/TestStackBangRbp.java times out on Solaris-Sparc V9
roland
parents:
23847
diff
changeset
|
7132 |
void MacroAssembler::verified_entry(int framesize, int stack_bang_size, bool fp_mode_24b) { |
14626 | 7133 |
|
7134 |
// WARNING: Initial instruction MUST be 5 bytes or longer so that |
|
7135 |
// NativeJump::patch_verified_entry will be able to patch out the entry |
|
7136 |
// code safely. The push to verify stack depth is ok at 5 bytes, |
|
7137 |
// the frame allocation can be either 3 or 6 bytes. So if we don't do |
|
7138 |
// stack bang then we must use the 6 byte frame allocation even if |
|
7139 |
// we have no frame. :-( |
|
24018
77b156916bab
8032410: compiler/uncommontrap/TestStackBangRbp.java times out on Solaris-Sparc V9
roland
parents:
23847
diff
changeset
|
7140 |
assert(stack_bang_size >= framesize || stack_bang_size <= 0, "stack bang size incorrect"); |
14626 | 7141 |
|
7142 |
assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned"); |
|
7143 |
// Remove word for return addr |
|
7144 |
framesize -= wordSize; |
|
24018
77b156916bab
8032410: compiler/uncommontrap/TestStackBangRbp.java times out on Solaris-Sparc V9
roland
parents:
23847
diff
changeset
|
7145 |
stack_bang_size -= wordSize; |
14626 | 7146 |
|
7147 |
// Calls to C2R adapters often do not accept exceptional returns. |
|
7148 |
// We require that their callers must bang for them. But be careful, because |
|
7149 |
// some VM calls (such as call site linkage) can use several kilobytes of |
|
7150 |
// stack. But the stack safety zone should account for that. |
|
7151 |
// See bugs 4446381, 4468289, 4497237. |
|
24018
77b156916bab
8032410: compiler/uncommontrap/TestStackBangRbp.java times out on Solaris-Sparc V9
roland
parents:
23847
diff
changeset
|
7152 |
if (stack_bang_size > 0) { |
77b156916bab
8032410: compiler/uncommontrap/TestStackBangRbp.java times out on Solaris-Sparc V9
roland
parents:
23847
diff
changeset
|
7153 |
generate_stack_overflow_check(stack_bang_size); |
14626 | 7154 |
|
7155 |
// We always push rbp, so that on return to interpreter rbp, will be |
|
7156 |
// restored correctly and we can correct the stack. |
|
7157 |
push(rbp); |
|
30305
b92a97e1e9cb
8068945: Use RBP register as proper frame pointer in JIT compiled code on x86
zmajo
parents:
30299
diff
changeset
|
7158 |
// Save caller's stack pointer into RBP if the frame pointer is preserved. |
b92a97e1e9cb
8068945: Use RBP register as proper frame pointer in JIT compiled code on x86
zmajo
parents:
30299
diff
changeset
|
7159 |
if (PreserveFramePointer) { |
b92a97e1e9cb
8068945: Use RBP register as proper frame pointer in JIT compiled code on x86
zmajo
parents:
30299
diff
changeset
|
7160 |
mov(rbp, rsp); |
b92a97e1e9cb
8068945: Use RBP register as proper frame pointer in JIT compiled code on x86
zmajo
parents:
30299
diff
changeset
|
7161 |
} |
14626 | 7162 |
// Remove word for ebp |
7163 |
framesize -= wordSize; |
|
7164 |
||
7165 |
// Create frame |
|
7166 |
if (framesize) { |
|
7167 |
subptr(rsp, framesize); |
|
7168 |
} |
|
7169 |
} else { |
|
7170 |
// Create frame (force generation of a 4 byte immediate value) |
|
7171 |
subptr_imm32(rsp, framesize); |
|
7172 |
||
7173 |
// Save RBP register now. |
|
7174 |
framesize -= wordSize; |
|
7175 |
movptr(Address(rsp, framesize), rbp); |
|
30305
b92a97e1e9cb
8068945: Use RBP register as proper frame pointer in JIT compiled code on x86
zmajo
parents:
30299
diff
changeset
|
7176 |
// Save caller's stack pointer into RBP if the frame pointer is preserved. |
b92a97e1e9cb
8068945: Use RBP register as proper frame pointer in JIT compiled code on x86
zmajo
parents:
30299
diff
changeset
|
7177 |
if (PreserveFramePointer) { |
b92a97e1e9cb
8068945: Use RBP register as proper frame pointer in JIT compiled code on x86
zmajo
parents:
30299
diff
changeset
|
7178 |
movptr(rbp, rsp); |
33188
6d91a3077eca
8080650: Enable stubs to use frame pointers correctly
zmajo
parents:
33160
diff
changeset
|
7179 |
if (framesize > 0) { |
6d91a3077eca
8080650: Enable stubs to use frame pointers correctly
zmajo
parents:
33160
diff
changeset
|
7180 |
addptr(rbp, framesize); |
6d91a3077eca
8080650: Enable stubs to use frame pointers correctly
zmajo
parents:
33160
diff
changeset
|
7181 |
} |
30305
b92a97e1e9cb
8068945: Use RBP register as proper frame pointer in JIT compiled code on x86
zmajo
parents:
30299
diff
changeset
|
7182 |
} |
14626 | 7183 |
} |
7184 |
||
7185 |
if (VerifyStackAtCalls) { // Majik cookie to verify stack depth |
|
7186 |
framesize -= wordSize; |
|
7187 |
movptr(Address(rsp, framesize), (int32_t)0xbadb100d); |
|
7188 |
} |
|
7189 |
||
7190 |
#ifndef _LP64 |
|
7191 |
// If method sets FPU control word do it now |
|
7192 |
if (fp_mode_24b) { |
|
7193 |
fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24())); |
|
7194 |
} |
|
7195 |
if (UseSSE >= 2 && VerifyFPU) { |
|
7196 |
verify_FPU(0, "FPU stack must be clean on entry"); |
|
7197 |
} |
|
7198 |
#endif |
|
7199 |
||
7200 |
#ifdef ASSERT |
|
7201 |
if (VerifyStackAtCalls) { |
|
7202 |
Label L; |
|
7203 |
push(rax); |
|
7204 |
mov(rax, rsp); |
|
7205 |
andptr(rax, StackAlignmentInBytes-1); |
|
7206 |
cmpptr(rax, StackAlignmentInBytes-wordSize); |
|
7207 |
pop(rax); |
|
7208 |
jcc(Assembler::equal, L); |
|
7209 |
STOP("Stack is not properly aligned!"); |
|
7210 |
bind(L); |
|
7211 |
} |
|
7212 |
#endif |
|
7213 |
||
7214 |
} |
|
7215 |
||
36554
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7216 |
void MacroAssembler::clear_mem(Register base, Register cnt, Register tmp, bool is_large) { |
15114
4074553c678b
8005522: use fast-string instructions on x86 for zeroing
kvn
parents:
14837
diff
changeset
|
7217 |
// cnt - number of qwords (8-byte words). |
4074553c678b
8005522: use fast-string instructions on x86 for zeroing
kvn
parents:
14837
diff
changeset
|
7218 |
// base - start address, qword aligned. |
36554
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7219 |
// is_large - if optimizers know cnt is larger than InitArrayShortSize |
15114
4074553c678b
8005522: use fast-string instructions on x86 for zeroing
kvn
parents:
14837
diff
changeset
|
7220 |
assert(base==rdi, "base register must be edi for rep stos"); |
4074553c678b
8005522: use fast-string instructions on x86 for zeroing
kvn
parents:
14837
diff
changeset
|
7221 |
assert(tmp==rax, "tmp register must be eax for rep stos"); |
4074553c678b
8005522: use fast-string instructions on x86 for zeroing
kvn
parents:
14837
diff
changeset
|
7222 |
assert(cnt==rcx, "cnt register must be ecx for rep stos"); |
36554
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7223 |
assert(InitArrayShortSize % BytesPerLong == 0, |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7224 |
"InitArrayShortSize should be the multiple of BytesPerLong"); |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7225 |
|
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7226 |
Label DONE; |
15114
4074553c678b
8005522: use fast-string instructions on x86 for zeroing
kvn
parents:
14837
diff
changeset
|
7227 |
|
4074553c678b
8005522: use fast-string instructions on x86 for zeroing
kvn
parents:
14837
diff
changeset
|
7228 |
xorptr(tmp, tmp); |
36554
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7229 |
|
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7230 |
if (!is_large) { |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7231 |
Label LOOP, LONG; |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7232 |
cmpptr(cnt, InitArrayShortSize/BytesPerLong); |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7233 |
jccb(Assembler::greater, LONG); |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7234 |
|
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7235 |
NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7236 |
|
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7237 |
decrement(cnt); |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7238 |
jccb(Assembler::negative, DONE); // Zero length |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7239 |
|
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7240 |
// Use individual pointer-sized stores for small counts: |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7241 |
BIND(LOOP); |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7242 |
movptr(Address(base, cnt, Address::times_ptr), tmp); |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7243 |
decrement(cnt); |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7244 |
jccb(Assembler::greaterEqual, LOOP); |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7245 |
jmpb(DONE); |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7246 |
|
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7247 |
BIND(LONG); |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7248 |
} |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7249 |
|
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7250 |
// Use longer rep-prefixed ops for non-small counts: |
15114
4074553c678b
8005522: use fast-string instructions on x86 for zeroing
kvn
parents:
14837
diff
changeset
|
7251 |
if (UseFastStosb) { |
36554
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7252 |
shlptr(cnt, 3); // convert to number of bytes |
15114
4074553c678b
8005522: use fast-string instructions on x86 for zeroing
kvn
parents:
14837
diff
changeset
|
7253 |
rep_stosb(); |
4074553c678b
8005522: use fast-string instructions on x86 for zeroing
kvn
parents:
14837
diff
changeset
|
7254 |
} else { |
36554
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7255 |
NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM |
15114
4074553c678b
8005522: use fast-string instructions on x86 for zeroing
kvn
parents:
14837
diff
changeset
|
7256 |
rep_stos(); |
4074553c678b
8005522: use fast-string instructions on x86 for zeroing
kvn
parents:
14837
diff
changeset
|
7257 |
} |
36554
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7258 |
|
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
7259 |
BIND(DONE); |
15114
4074553c678b
8005522: use fast-string instructions on x86 for zeroing
kvn
parents:
14837
diff
changeset
|
7260 |
} |
14626 | 7261 |
|
33628 | 7262 |
#ifdef COMPILER2 |
7263 |
||
14626 | 7264 |
// IndexOf for constant substrings with size >= 8 chars |
7265 |
// which don't need to be loaded through stack. |
|
7266 |
void MacroAssembler::string_indexofC8(Register str1, Register str2, |
|
7267 |
Register cnt1, Register cnt2, |
|
7268 |
int int_cnt2, Register result, |
|
33628 | 7269 |
XMMRegister vec, Register tmp, |
7270 |
int ae) { |
|
14626 | 7271 |
ShortBranchVerifier sbv(this); |
34207
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7272 |
assert(UseSSE42Intrinsics, "SSE4.2 intrinsics are required"); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7273 |
assert(UseSSE >= 4, "SSE4 must be enabled for SSE4.2 intrinsics to be available"); |
33628 | 7274 |
assert(ae != StrIntrinsicNode::LU, "Invalid encoding"); |
7275 |
||
7276 |
// This method uses the pcmpestri instruction with bound registers |
|
14626 | 7277 |
// inputs: |
7278 |
// xmm - substring |
|
7279 |
// rax - substring length (elements count) |
|
7280 |
// mem - scanned string |
|
7281 |
// rdx - string length (elements count) |
|
7282 |
// 0xd - mode: 1100 (substring search) + 01 (unsigned shorts) |
|
33628 | 7283 |
// 0xc - mode: 1100 (substring search) + 00 (unsigned bytes) |
14626 | 7284 |
// outputs: |
7285 |
// rcx - matched index in string |
|
7286 |
assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri"); |
|
33628 | 7287 |
int mode = (ae == StrIntrinsicNode::LL) ? 0x0c : 0x0d; // bytes or shorts |
7288 |
int stride = (ae == StrIntrinsicNode::LL) ? 16 : 8; //UU, UL -> 8 |
|
7289 |
Address::ScaleFactor scale1 = (ae == StrIntrinsicNode::LL) ? Address::times_1 : Address::times_2; |
|
7290 |
Address::ScaleFactor scale2 = (ae == StrIntrinsicNode::UL) ? Address::times_1 : scale1; |
|
14626 | 7291 |
|
7292 |
Label RELOAD_SUBSTR, SCAN_TO_SUBSTR, SCAN_SUBSTR, |
|
7293 |
RET_FOUND, RET_NOT_FOUND, EXIT, FOUND_SUBSTR, |
|
7294 |
MATCH_SUBSTR_HEAD, RELOAD_STR, FOUND_CANDIDATE; |
|
7295 |
||
7296 |
// Note, inline_string_indexOf() generates checks: |
|
7297 |
// if (substr.count > string.count) return -1; |
|
7298 |
// if (substr.count == 0) return 0; |
|
33628 | 7299 |
assert(int_cnt2 >= stride, "this code is used only for cnt2 >= 8 chars"); |
14626 | 7300 |
|
7301 |
// Load substring. |
|
33628 | 7302 |
if (ae == StrIntrinsicNode::UL) { |
7303 |
pmovzxbw(vec, Address(str2, 0)); |
|
7304 |
} else { |
|
7305 |
movdqu(vec, Address(str2, 0)); |
|
7306 |
} |
|
14626 | 7307 |
movl(cnt2, int_cnt2); |
7308 |
movptr(result, str1); // string addr |
|
7309 |
||
33628 | 7310 |
if (int_cnt2 > stride) { |
14626 | 7311 |
jmpb(SCAN_TO_SUBSTR); |
7312 |
||
7313 |
// Reload substr for rescan, this code |
|
7314 |
// is executed only for large substrings (> 8 chars) |
|
7315 |
bind(RELOAD_SUBSTR); |
|
33628 | 7316 |
if (ae == StrIntrinsicNode::UL) { |
7317 |
pmovzxbw(vec, Address(str2, 0)); |
|
7318 |
} else { |
|
7319 |
movdqu(vec, Address(str2, 0)); |
|
7320 |
} |
|
14626 | 7321 |
negptr(cnt2); // Jumped here with negative cnt2, convert to positive |
7322 |
||
7323 |
bind(RELOAD_STR); |
|
7324 |
// We came here after the beginning of the substring was |
|
7325 |
// matched but the rest of it was not so we need to search |
|
7326 |
// again. Start from the next element after the previous match. |
|
7327 |
||
7328 |
// cnt2 is number of substring reminding elements and |
|
7329 |
// cnt1 is number of string reminding elements when cmp failed. |
|
7330 |
// Restored cnt1 = cnt1 - cnt2 + int_cnt2 |
|
7331 |
subl(cnt1, cnt2); |
|
7332 |
addl(cnt1, int_cnt2); |
|
7333 |
movl(cnt2, int_cnt2); // Now restore cnt2 |
|
7334 |
||
7335 |
decrementl(cnt1); // Shift to next element |
|
7336 |
cmpl(cnt1, cnt2); |
|
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
7337 |
jcc(Assembler::negative, RET_NOT_FOUND); // Left less then substring |
14626 | 7338 |
|
33628 | 7339 |
addptr(result, (1<<scale1)); |
14626 | 7340 |
|
7341 |
} // (int_cnt2 > 8) |
|
7342 |
||
7343 |
// Scan string for start of substr in 16-byte vectors |
|
7344 |
bind(SCAN_TO_SUBSTR); |
|
33628 | 7345 |
pcmpestri(vec, Address(result, 0), mode); |
14626 | 7346 |
jccb(Assembler::below, FOUND_CANDIDATE); // CF == 1 |
33628 | 7347 |
subl(cnt1, stride); |
14626 | 7348 |
jccb(Assembler::lessEqual, RET_NOT_FOUND); // Scanned full string |
7349 |
cmpl(cnt1, cnt2); |
|
7350 |
jccb(Assembler::negative, RET_NOT_FOUND); // Left less then substring |
|
7351 |
addptr(result, 16); |
|
7352 |
jmpb(SCAN_TO_SUBSTR); |
|
7353 |
||
7354 |
// Found a potential substr |
|
7355 |
bind(FOUND_CANDIDATE); |
|
7356 |
// Matched whole vector if first element matched (tmp(rcx) == 0). |
|
33628 | 7357 |
if (int_cnt2 == stride) { |
14626 | 7358 |
jccb(Assembler::overflow, RET_FOUND); // OF == 1 |
7359 |
} else { // int_cnt2 > 8 |
|
7360 |
jccb(Assembler::overflow, FOUND_SUBSTR); |
|
7361 |
} |
|
7362 |
// After pcmpestri tmp(rcx) contains matched element index |
|
7363 |
// Compute start addr of substr |
|
33628 | 7364 |
lea(result, Address(result, tmp, scale1)); |
14626 | 7365 |
|
7366 |
// Make sure string is still long enough |
|
7367 |
subl(cnt1, tmp); |
|
7368 |
cmpl(cnt1, cnt2); |
|
33628 | 7369 |
if (int_cnt2 == stride) { |
14626 | 7370 |
jccb(Assembler::greaterEqual, SCAN_TO_SUBSTR); |
7371 |
} else { // int_cnt2 > 8 |
|
7372 |
jccb(Assembler::greaterEqual, MATCH_SUBSTR_HEAD); |
|
7373 |
} |
|
7374 |
// Left less then substring. |
|
7375 |
||
7376 |
bind(RET_NOT_FOUND); |
|
7377 |
movl(result, -1); |
|
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
7378 |
jmp(EXIT); |
14626 | 7379 |
|
33628 | 7380 |
if (int_cnt2 > stride) { |
14626 | 7381 |
// This code is optimized for the case when whole substring |
7382 |
// is matched if its head is matched. |
|
7383 |
bind(MATCH_SUBSTR_HEAD); |
|
33628 | 7384 |
pcmpestri(vec, Address(result, 0), mode); |
14626 | 7385 |
// Reload only string if does not match |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
7386 |
jcc(Assembler::noOverflow, RELOAD_STR); // OF == 0 |
14626 | 7387 |
|
7388 |
Label CONT_SCAN_SUBSTR; |
|
7389 |
// Compare the rest of substring (> 8 chars). |
|
7390 |
bind(FOUND_SUBSTR); |
|
7391 |
// First 8 chars are already matched. |
|
7392 |
negptr(cnt2); |
|
33628 | 7393 |
addptr(cnt2, stride); |
14626 | 7394 |
|
7395 |
bind(SCAN_SUBSTR); |
|
33628 | 7396 |
subl(cnt1, stride); |
7397 |
cmpl(cnt2, -stride); // Do not read beyond substring |
|
14626 | 7398 |
jccb(Assembler::lessEqual, CONT_SCAN_SUBSTR); |
7399 |
// Back-up strings to avoid reading beyond substring: |
|
7400 |
// cnt1 = cnt1 - cnt2 + 8 |
|
7401 |
addl(cnt1, cnt2); // cnt2 is negative |
|
33628 | 7402 |
addl(cnt1, stride); |
7403 |
movl(cnt2, stride); negptr(cnt2); |
|
14626 | 7404 |
bind(CONT_SCAN_SUBSTR); |
7405 |
if (int_cnt2 < (int)G) { |
|
33628 | 7406 |
int tail_off1 = int_cnt2<<scale1; |
7407 |
int tail_off2 = int_cnt2<<scale2; |
|
7408 |
if (ae == StrIntrinsicNode::UL) { |
|
7409 |
pmovzxbw(vec, Address(str2, cnt2, scale2, tail_off2)); |
|
7410 |
} else { |
|
7411 |
movdqu(vec, Address(str2, cnt2, scale2, tail_off2)); |
|
7412 |
} |
|
7413 |
pcmpestri(vec, Address(result, cnt2, scale1, tail_off1), mode); |
|
14626 | 7414 |
} else { |
7415 |
// calculate index in register to avoid integer overflow (int_cnt2*2) |
|
7416 |
movl(tmp, int_cnt2); |
|
7417 |
addptr(tmp, cnt2); |
|
33628 | 7418 |
if (ae == StrIntrinsicNode::UL) { |
7419 |
pmovzxbw(vec, Address(str2, tmp, scale2, 0)); |
|
7420 |
} else { |
|
7421 |
movdqu(vec, Address(str2, tmp, scale2, 0)); |
|
7422 |
} |
|
7423 |
pcmpestri(vec, Address(result, tmp, scale1, 0), mode); |
|
14626 | 7424 |
} |
7425 |
// Need to reload strings pointers if not matched whole vector |
|
7426 |
jcc(Assembler::noOverflow, RELOAD_SUBSTR); // OF == 0 |
|
33628 | 7427 |
addptr(cnt2, stride); |
14626 | 7428 |
jcc(Assembler::negative, SCAN_SUBSTR); |
7429 |
// Fall through if found full substring |
|
7430 |
||
7431 |
} // (int_cnt2 > 8) |
|
7432 |
||
7433 |
bind(RET_FOUND); |
|
7434 |
// Found result if we matched full small substring. |
|
7435 |
// Compute substr offset |
|
7436 |
subptr(result, str1); |
|
33628 | 7437 |
if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) { |
7438 |
shrl(result, 1); // index |
|
7439 |
} |
|
14626 | 7440 |
bind(EXIT); |
7441 |
||
7442 |
} // string_indexofC8 |
|
7443 |
||
7444 |
// Small strings are loaded through stack if they cross page boundary. |
|
7445 |
void MacroAssembler::string_indexof(Register str1, Register str2, |
|
7446 |
Register cnt1, Register cnt2, |
|
7447 |
int int_cnt2, Register result, |
|
33628 | 7448 |
XMMRegister vec, Register tmp, |
7449 |
int ae) { |
|
14626 | 7450 |
ShortBranchVerifier sbv(this); |
34207
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7451 |
assert(UseSSE42Intrinsics, "SSE4.2 intrinsics are required"); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7452 |
assert(UseSSE >= 4, "SSE4 must be enabled for SSE4.2 intrinsics to be available"); |
33628 | 7453 |
assert(ae != StrIntrinsicNode::LU, "Invalid encoding"); |
7454 |
||
14626 | 7455 |
// |
7456 |
// int_cnt2 is length of small (< 8 chars) constant substring |
|
7457 |
// or (-1) for non constant substring in which case its length |
|
7458 |
// is in cnt2 register. |
|
7459 |
// |
|
7460 |
// Note, inline_string_indexOf() generates checks: |
|
7461 |
// if (substr.count > string.count) return -1; |
|
7462 |
// if (substr.count == 0) return 0; |
|
7463 |
// |
|
33628 | 7464 |
int stride = (ae == StrIntrinsicNode::LL) ? 16 : 8; //UU, UL -> 8 |
7465 |
assert(int_cnt2 == -1 || (0 < int_cnt2 && int_cnt2 < stride), "should be != 0"); |
|
7466 |
// This method uses the pcmpestri instruction with bound registers |
|
14626 | 7467 |
// inputs: |
7468 |
// xmm - substring |
|
7469 |
// rax - substring length (elements count) |
|
7470 |
// mem - scanned string |
|
7471 |
// rdx - string length (elements count) |
|
7472 |
// 0xd - mode: 1100 (substring search) + 01 (unsigned shorts) |
|
33628 | 7473 |
// 0xc - mode: 1100 (substring search) + 00 (unsigned bytes) |
14626 | 7474 |
// outputs: |
7475 |
// rcx - matched index in string |
|
7476 |
assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri"); |
|
33628 | 7477 |
int mode = (ae == StrIntrinsicNode::LL) ? 0x0c : 0x0d; // bytes or shorts |
7478 |
Address::ScaleFactor scale1 = (ae == StrIntrinsicNode::LL) ? Address::times_1 : Address::times_2; |
|
7479 |
Address::ScaleFactor scale2 = (ae == StrIntrinsicNode::UL) ? Address::times_1 : scale1; |
|
14626 | 7480 |
|
7481 |
Label RELOAD_SUBSTR, SCAN_TO_SUBSTR, SCAN_SUBSTR, ADJUST_STR, |
|
7482 |
RET_FOUND, RET_NOT_FOUND, CLEANUP, FOUND_SUBSTR, |
|
7483 |
FOUND_CANDIDATE; |
|
7484 |
||
7485 |
{ //======================================================== |
|
7486 |
// We don't know where these strings are located |
|
7487 |
// and we can't read beyond them. Load them through stack. |
|
7488 |
Label BIG_STRINGS, CHECK_STR, COPY_SUBSTR, COPY_STR; |
|
7489 |
||
7490 |
movptr(tmp, rsp); // save old SP |
|
7491 |
||
7492 |
if (int_cnt2 > 0) { // small (< 8 chars) constant substring |
|
33628 | 7493 |
if (int_cnt2 == (1>>scale2)) { // One byte |
7494 |
assert((ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL), "Only possible for latin1 encoding"); |
|
7495 |
load_unsigned_byte(result, Address(str2, 0)); |
|
7496 |
movdl(vec, result); // move 32 bits |
|
7497 |
} else if (ae == StrIntrinsicNode::LL && int_cnt2 == 3) { // Three bytes |
|
7498 |
// Not enough header space in 32-bit VM: 12+3 = 15. |
|
7499 |
movl(result, Address(str2, -1)); |
|
7500 |
shrl(result, 8); |
|
7501 |
movdl(vec, result); // move 32 bits |
|
7502 |
} else if (ae != StrIntrinsicNode::UL && int_cnt2 == (2>>scale2)) { // One char |
|
14626 | 7503 |
load_unsigned_short(result, Address(str2, 0)); |
7504 |
movdl(vec, result); // move 32 bits |
|
33628 | 7505 |
} else if (ae != StrIntrinsicNode::UL && int_cnt2 == (4>>scale2)) { // Two chars |
14626 | 7506 |
movdl(vec, Address(str2, 0)); // move 32 bits |
33628 | 7507 |
} else if (ae != StrIntrinsicNode::UL && int_cnt2 == (8>>scale2)) { // Four chars |
14626 | 7508 |
movq(vec, Address(str2, 0)); // move 64 bits |
33628 | 7509 |
} else { // cnt2 = { 3, 5, 6, 7 } || (ae == StrIntrinsicNode::UL && cnt2 ={2, ..., 7}) |
14626 | 7510 |
// Array header size is 12 bytes in 32-bit VM |
7511 |
// + 6 bytes for 3 chars == 18 bytes, |
|
7512 |
// enough space to load vec and shift. |
|
7513 |
assert(HeapWordSize*TypeArrayKlass::header_size() >= 12,"sanity"); |
|
33628 | 7514 |
if (ae == StrIntrinsicNode::UL) { |
7515 |
int tail_off = int_cnt2-8; |
|
7516 |
pmovzxbw(vec, Address(str2, tail_off)); |
|
7517 |
psrldq(vec, -2*tail_off); |
|
7518 |
} |
|
7519 |
else { |
|
7520 |
int tail_off = int_cnt2*(1<<scale2); |
|
7521 |
movdqu(vec, Address(str2, tail_off-16)); |
|
7522 |
psrldq(vec, 16-tail_off); |
|
7523 |
} |
|
14626 | 7524 |
} |
7525 |
} else { // not constant substring |
|
33628 | 7526 |
cmpl(cnt2, stride); |
14626 | 7527 |
jccb(Assembler::aboveEqual, BIG_STRINGS); // Both strings are big enough |
7528 |
||
7529 |
// We can read beyond string if srt+16 does not cross page boundary |
|
7530 |
// since heaps are aligned and mapped by pages. |
|
7531 |
assert(os::vm_page_size() < (int)G, "default page should be small"); |
|
7532 |
movl(result, str2); // We need only low 32 bits |
|
7533 |
andl(result, (os::vm_page_size()-1)); |
|
7534 |
cmpl(result, (os::vm_page_size()-16)); |
|
7535 |
jccb(Assembler::belowEqual, CHECK_STR); |
|
7536 |
||
7537 |
// Move small strings to stack to allow load 16 bytes into vec. |
|
7538 |
subptr(rsp, 16); |
|
33628 | 7539 |
int stk_offset = wordSize-(1<<scale2); |
14626 | 7540 |
push(cnt2); |
7541 |
||
7542 |
bind(COPY_SUBSTR); |
|
33628 | 7543 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL) { |
7544 |
load_unsigned_byte(result, Address(str2, cnt2, scale2, -1)); |
|
7545 |
movb(Address(rsp, cnt2, scale2, stk_offset), result); |
|
7546 |
} else if (ae == StrIntrinsicNode::UU) { |
|
7547 |
load_unsigned_short(result, Address(str2, cnt2, scale2, -2)); |
|
7548 |
movw(Address(rsp, cnt2, scale2, stk_offset), result); |
|
7549 |
} |
|
14626 | 7550 |
decrement(cnt2); |
7551 |
jccb(Assembler::notZero, COPY_SUBSTR); |
|
7552 |
||
7553 |
pop(cnt2); |
|
7554 |
movptr(str2, rsp); // New substring address |
|
7555 |
} // non constant |
|
7556 |
||
7557 |
bind(CHECK_STR); |
|
33628 | 7558 |
cmpl(cnt1, stride); |
14626 | 7559 |
jccb(Assembler::aboveEqual, BIG_STRINGS); |
7560 |
||
7561 |
// Check cross page boundary. |
|
7562 |
movl(result, str1); // We need only low 32 bits |
|
7563 |
andl(result, (os::vm_page_size()-1)); |
|
7564 |
cmpl(result, (os::vm_page_size()-16)); |
|
7565 |
jccb(Assembler::belowEqual, BIG_STRINGS); |
|
7566 |
||
7567 |
subptr(rsp, 16); |
|
33628 | 7568 |
int stk_offset = -(1<<scale1); |
14626 | 7569 |
if (int_cnt2 < 0) { // not constant |
7570 |
push(cnt2); |
|
7571 |
stk_offset += wordSize; |
|
7572 |
} |
|
7573 |
movl(cnt2, cnt1); |
|
7574 |
||
7575 |
bind(COPY_STR); |
|
33628 | 7576 |
if (ae == StrIntrinsicNode::LL) { |
7577 |
load_unsigned_byte(result, Address(str1, cnt2, scale1, -1)); |
|
7578 |
movb(Address(rsp, cnt2, scale1, stk_offset), result); |
|
7579 |
} else { |
|
7580 |
load_unsigned_short(result, Address(str1, cnt2, scale1, -2)); |
|
7581 |
movw(Address(rsp, cnt2, scale1, stk_offset), result); |
|
7582 |
} |
|
14626 | 7583 |
decrement(cnt2); |
7584 |
jccb(Assembler::notZero, COPY_STR); |
|
7585 |
||
7586 |
if (int_cnt2 < 0) { // not constant |
|
7587 |
pop(cnt2); |
|
7588 |
} |
|
7589 |
movptr(str1, rsp); // New string address |
|
7590 |
||
7591 |
bind(BIG_STRINGS); |
|
7592 |
// Load substring. |
|
7593 |
if (int_cnt2 < 0) { // -1 |
|
33628 | 7594 |
if (ae == StrIntrinsicNode::UL) { |
7595 |
pmovzxbw(vec, Address(str2, 0)); |
|
7596 |
} else { |
|
7597 |
movdqu(vec, Address(str2, 0)); |
|
7598 |
} |
|
14626 | 7599 |
push(cnt2); // substr count |
7600 |
push(str2); // substr addr |
|
7601 |
push(str1); // string addr |
|
7602 |
} else { |
|
7603 |
// Small (< 8 chars) constant substrings are loaded already. |
|
7604 |
movl(cnt2, int_cnt2); |
|
7605 |
} |
|
7606 |
push(tmp); // original SP |
|
7607 |
||
7608 |
} // Finished loading |
|
7609 |
||
7610 |
//======================================================== |
|
7611 |
// Start search |
|
7612 |
// |
|
7613 |
||
7614 |
movptr(result, str1); // string addr |
|
7615 |
||
7616 |
if (int_cnt2 < 0) { // Only for non constant substring |
|
7617 |
jmpb(SCAN_TO_SUBSTR); |
|
7618 |
||
7619 |
// SP saved at sp+0 |
|
7620 |
// String saved at sp+1*wordSize |
|
7621 |
// Substr saved at sp+2*wordSize |
|
7622 |
// Substr count saved at sp+3*wordSize |
|
7623 |
||
7624 |
// Reload substr for rescan, this code |
|
7625 |
// is executed only for large substrings (> 8 chars) |
|
7626 |
bind(RELOAD_SUBSTR); |
|
7627 |
movptr(str2, Address(rsp, 2*wordSize)); |
|
7628 |
movl(cnt2, Address(rsp, 3*wordSize)); |
|
33628 | 7629 |
if (ae == StrIntrinsicNode::UL) { |
7630 |
pmovzxbw(vec, Address(str2, 0)); |
|
7631 |
} else { |
|
7632 |
movdqu(vec, Address(str2, 0)); |
|
7633 |
} |
|
14626 | 7634 |
// We came here after the beginning of the substring was |
7635 |
// matched but the rest of it was not so we need to search |
|
7636 |
// again. Start from the next element after the previous match. |
|
7637 |
subptr(str1, result); // Restore counter |
|
33628 | 7638 |
if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) { |
7639 |
shrl(str1, 1); |
|
7640 |
} |
|
14626 | 7641 |
addl(cnt1, str1); |
7642 |
decrementl(cnt1); // Shift to next element |
|
7643 |
cmpl(cnt1, cnt2); |
|
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
7644 |
jcc(Assembler::negative, RET_NOT_FOUND); // Left less then substring |
14626 | 7645 |
|
33628 | 7646 |
addptr(result, (1<<scale1)); |
14626 | 7647 |
} // non constant |
7648 |
||
7649 |
// Scan string for start of substr in 16-byte vectors |
|
7650 |
bind(SCAN_TO_SUBSTR); |
|
7651 |
assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri"); |
|
33628 | 7652 |
pcmpestri(vec, Address(result, 0), mode); |
14626 | 7653 |
jccb(Assembler::below, FOUND_CANDIDATE); // CF == 1 |
33628 | 7654 |
subl(cnt1, stride); |
14626 | 7655 |
jccb(Assembler::lessEqual, RET_NOT_FOUND); // Scanned full string |
7656 |
cmpl(cnt1, cnt2); |
|
7657 |
jccb(Assembler::negative, RET_NOT_FOUND); // Left less then substring |
|
7658 |
addptr(result, 16); |
|
7659 |
||
7660 |
bind(ADJUST_STR); |
|
33628 | 7661 |
cmpl(cnt1, stride); // Do not read beyond string |
14626 | 7662 |
jccb(Assembler::greaterEqual, SCAN_TO_SUBSTR); |
7663 |
// Back-up string to avoid reading beyond string. |
|
33628 | 7664 |
lea(result, Address(result, cnt1, scale1, -16)); |
7665 |
movl(cnt1, stride); |
|
14626 | 7666 |
jmpb(SCAN_TO_SUBSTR); |
7667 |
||
7668 |
// Found a potential substr |
|
7669 |
bind(FOUND_CANDIDATE); |
|
7670 |
// After pcmpestri tmp(rcx) contains matched element index |
|
7671 |
||
7672 |
// Make sure string is still long enough |
|
7673 |
subl(cnt1, tmp); |
|
7674 |
cmpl(cnt1, cnt2); |
|
7675 |
jccb(Assembler::greaterEqual, FOUND_SUBSTR); |
|
7676 |
// Left less then substring. |
|
7677 |
||
7678 |
bind(RET_NOT_FOUND); |
|
7679 |
movl(result, -1); |
|
7680 |
jmpb(CLEANUP); |
|
7681 |
||
7682 |
bind(FOUND_SUBSTR); |
|
7683 |
// Compute start addr of substr |
|
33628 | 7684 |
lea(result, Address(result, tmp, scale1)); |
14626 | 7685 |
if (int_cnt2 > 0) { // Constant substring |
7686 |
// Repeat search for small substring (< 8 chars) |
|
7687 |
// from new point without reloading substring. |
|
7688 |
// Have to check that we don't read beyond string. |
|
33628 | 7689 |
cmpl(tmp, stride-int_cnt2); |
14626 | 7690 |
jccb(Assembler::greater, ADJUST_STR); |
7691 |
// Fall through if matched whole substring. |
|
7692 |
} else { // non constant |
|
7693 |
assert(int_cnt2 == -1, "should be != 0"); |
|
7694 |
||
7695 |
addl(tmp, cnt2); |
|
7696 |
// Found result if we matched whole substring. |
|
33628 | 7697 |
cmpl(tmp, stride); |
14626 | 7698 |
jccb(Assembler::lessEqual, RET_FOUND); |
7699 |
||
7700 |
// Repeat search for small substring (<= 8 chars) |
|
7701 |
// from new point 'str1' without reloading substring. |
|
33628 | 7702 |
cmpl(cnt2, stride); |
14626 | 7703 |
// Have to check that we don't read beyond string. |
7704 |
jccb(Assembler::lessEqual, ADJUST_STR); |
|
7705 |
||
7706 |
Label CHECK_NEXT, CONT_SCAN_SUBSTR, RET_FOUND_LONG; |
|
7707 |
// Compare the rest of substring (> 8 chars). |
|
7708 |
movptr(str1, result); |
|
7709 |
||
7710 |
cmpl(tmp, cnt2); |
|
7711 |
// First 8 chars are already matched. |
|
7712 |
jccb(Assembler::equal, CHECK_NEXT); |
|
7713 |
||
7714 |
bind(SCAN_SUBSTR); |
|
33628 | 7715 |
pcmpestri(vec, Address(str1, 0), mode); |
14626 | 7716 |
// Need to reload strings pointers if not matched whole vector |
7717 |
jcc(Assembler::noOverflow, RELOAD_SUBSTR); // OF == 0 |
|
7718 |
||
7719 |
bind(CHECK_NEXT); |
|
33628 | 7720 |
subl(cnt2, stride); |
14626 | 7721 |
jccb(Assembler::lessEqual, RET_FOUND_LONG); // Found full substring |
7722 |
addptr(str1, 16); |
|
33628 | 7723 |
if (ae == StrIntrinsicNode::UL) { |
7724 |
addptr(str2, 8); |
|
7725 |
} else { |
|
7726 |
addptr(str2, 16); |
|
7727 |
} |
|
7728 |
subl(cnt1, stride); |
|
7729 |
cmpl(cnt2, stride); // Do not read beyond substring |
|
14626 | 7730 |
jccb(Assembler::greaterEqual, CONT_SCAN_SUBSTR); |
7731 |
// Back-up strings to avoid reading beyond substring. |
|
33628 | 7732 |
|
7733 |
if (ae == StrIntrinsicNode::UL) { |
|
7734 |
lea(str2, Address(str2, cnt2, scale2, -8)); |
|
7735 |
lea(str1, Address(str1, cnt2, scale1, -16)); |
|
7736 |
} else { |
|
7737 |
lea(str2, Address(str2, cnt2, scale2, -16)); |
|
7738 |
lea(str1, Address(str1, cnt2, scale1, -16)); |
|
7739 |
} |
|
14626 | 7740 |
subl(cnt1, cnt2); |
33628 | 7741 |
movl(cnt2, stride); |
7742 |
addl(cnt1, stride); |
|
14626 | 7743 |
bind(CONT_SCAN_SUBSTR); |
33628 | 7744 |
if (ae == StrIntrinsicNode::UL) { |
7745 |
pmovzxbw(vec, Address(str2, 0)); |
|
7746 |
} else { |
|
7747 |
movdqu(vec, Address(str2, 0)); |
|
7748 |
} |
|
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
7749 |
jmp(SCAN_SUBSTR); |
14626 | 7750 |
|
7751 |
bind(RET_FOUND_LONG); |
|
7752 |
movptr(str1, Address(rsp, wordSize)); |
|
7753 |
} // non constant |
|
7754 |
||
7755 |
bind(RET_FOUND); |
|
7756 |
// Compute substr offset |
|
7757 |
subptr(result, str1); |
|
33628 | 7758 |
if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) { |
7759 |
shrl(result, 1); // index |
|
7760 |
} |
|
14626 | 7761 |
bind(CLEANUP); |
7762 |
pop(rsp); // restore SP |
|
7763 |
||
7764 |
} // string_indexof |
|
7765 |
||
33628 | 7766 |
void MacroAssembler::string_indexof_char(Register str1, Register cnt1, Register ch, Register result, |
7767 |
XMMRegister vec1, XMMRegister vec2, XMMRegister vec3, Register tmp) { |
|
7768 |
ShortBranchVerifier sbv(this); |
|
34207
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7769 |
assert(UseSSE42Intrinsics, "SSE4.2 intrinsics are required"); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7770 |
assert(UseSSE >= 4, "SSE4 must be enabled for SSE4.2 intrinsics to be available"); |
33628 | 7771 |
|
7772 |
int stride = 8; |
|
7773 |
||
7774 |
Label FOUND_CHAR, SCAN_TO_CHAR, SCAN_TO_CHAR_LOOP, |
|
7775 |
SCAN_TO_8_CHAR, SCAN_TO_8_CHAR_LOOP, SCAN_TO_16_CHAR_LOOP, |
|
7776 |
RET_NOT_FOUND, SCAN_TO_8_CHAR_INIT, |
|
7777 |
FOUND_SEQ_CHAR, DONE_LABEL; |
|
7778 |
||
7779 |
movptr(result, str1); |
|
7780 |
if (UseAVX >= 2) { |
|
7781 |
cmpl(cnt1, stride); |
|
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
7782 |
jcc(Assembler::less, SCAN_TO_CHAR_LOOP); |
33628 | 7783 |
cmpl(cnt1, 2*stride); |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
7784 |
jcc(Assembler::less, SCAN_TO_8_CHAR_INIT); |
33628 | 7785 |
movdl(vec1, ch); |
7786 |
vpbroadcastw(vec1, vec1); |
|
7787 |
vpxor(vec2, vec2); |
|
7788 |
movl(tmp, cnt1); |
|
7789 |
andl(tmp, 0xFFFFFFF0); //vector count (in chars) |
|
7790 |
andl(cnt1,0x0000000F); //tail count (in chars) |
|
7791 |
||
7792 |
bind(SCAN_TO_16_CHAR_LOOP); |
|
7793 |
vmovdqu(vec3, Address(result, 0)); |
|
34162 | 7794 |
vpcmpeqw(vec3, vec3, vec1, 1); |
33628 | 7795 |
vptest(vec2, vec3); |
7796 |
jcc(Assembler::carryClear, FOUND_CHAR); |
|
7797 |
addptr(result, 32); |
|
7798 |
subl(tmp, 2*stride); |
|
7799 |
jccb(Assembler::notZero, SCAN_TO_16_CHAR_LOOP); |
|
7800 |
jmp(SCAN_TO_8_CHAR); |
|
7801 |
bind(SCAN_TO_8_CHAR_INIT); |
|
7802 |
movdl(vec1, ch); |
|
7803 |
pshuflw(vec1, vec1, 0x00); |
|
7804 |
pshufd(vec1, vec1, 0); |
|
7805 |
pxor(vec2, vec2); |
|
7806 |
} |
|
34207
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7807 |
bind(SCAN_TO_8_CHAR); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7808 |
cmpl(cnt1, stride); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7809 |
if (UseAVX >= 2) { |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
7810 |
jcc(Assembler::less, SCAN_TO_CHAR); |
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
7811 |
} else { |
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
7812 |
jcc(Assembler::less, SCAN_TO_CHAR_LOOP); |
34207
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7813 |
movdl(vec1, ch); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7814 |
pshuflw(vec1, vec1, 0x00); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7815 |
pshufd(vec1, vec1, 0); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7816 |
pxor(vec2, vec2); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7817 |
} |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7818 |
movl(tmp, cnt1); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7819 |
andl(tmp, 0xFFFFFFF8); //vector count (in chars) |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7820 |
andl(cnt1,0x00000007); //tail count (in chars) |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7821 |
|
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7822 |
bind(SCAN_TO_8_CHAR_LOOP); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7823 |
movdqu(vec3, Address(result, 0)); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7824 |
pcmpeqw(vec3, vec1); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7825 |
ptest(vec2, vec3); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7826 |
jcc(Assembler::carryClear, FOUND_CHAR); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7827 |
addptr(result, 16); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7828 |
subl(tmp, stride); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7829 |
jccb(Assembler::notZero, SCAN_TO_8_CHAR_LOOP); |
33628 | 7830 |
bind(SCAN_TO_CHAR); |
7831 |
testl(cnt1, cnt1); |
|
7832 |
jcc(Assembler::zero, RET_NOT_FOUND); |
|
7833 |
bind(SCAN_TO_CHAR_LOOP); |
|
7834 |
load_unsigned_short(tmp, Address(result, 0)); |
|
7835 |
cmpl(ch, tmp); |
|
7836 |
jccb(Assembler::equal, FOUND_SEQ_CHAR); |
|
7837 |
addptr(result, 2); |
|
7838 |
subl(cnt1, 1); |
|
7839 |
jccb(Assembler::zero, RET_NOT_FOUND); |
|
7840 |
jmp(SCAN_TO_CHAR_LOOP); |
|
7841 |
||
7842 |
bind(RET_NOT_FOUND); |
|
7843 |
movl(result, -1); |
|
7844 |
jmpb(DONE_LABEL); |
|
7845 |
||
34207
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7846 |
bind(FOUND_CHAR); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7847 |
if (UseAVX >= 2) { |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7848 |
vpmovmskb(tmp, vec3); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7849 |
} else { |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7850 |
pmovmskb(tmp, vec3); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7851 |
} |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7852 |
bsfl(ch, tmp); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7853 |
addl(result, ch); |
33628 | 7854 |
|
7855 |
bind(FOUND_SEQ_CHAR); |
|
7856 |
subptr(result, str1); |
|
7857 |
shrl(result, 1); |
|
7858 |
||
7859 |
bind(DONE_LABEL); |
|
7860 |
} // string_indexof_char |
|
7861 |
||
7862 |
// helper function for string_compare |
|
7863 |
void MacroAssembler::load_next_elements(Register elem1, Register elem2, Register str1, Register str2, |
|
7864 |
Address::ScaleFactor scale, Address::ScaleFactor scale1, |
|
7865 |
Address::ScaleFactor scale2, Register index, int ae) { |
|
7866 |
if (ae == StrIntrinsicNode::LL) { |
|
7867 |
load_unsigned_byte(elem1, Address(str1, index, scale, 0)); |
|
7868 |
load_unsigned_byte(elem2, Address(str2, index, scale, 0)); |
|
7869 |
} else if (ae == StrIntrinsicNode::UU) { |
|
7870 |
load_unsigned_short(elem1, Address(str1, index, scale, 0)); |
|
7871 |
load_unsigned_short(elem2, Address(str2, index, scale, 0)); |
|
7872 |
} else { |
|
7873 |
load_unsigned_byte(elem1, Address(str1, index, scale1, 0)); |
|
7874 |
load_unsigned_short(elem2, Address(str2, index, scale2, 0)); |
|
7875 |
} |
|
7876 |
} |
|
7877 |
||
7878 |
// Compare strings, used for char[] and byte[]. |
|
14626 | 7879 |
void MacroAssembler::string_compare(Register str1, Register str2, |
7880 |
Register cnt1, Register cnt2, Register result, |
|
33628 | 7881 |
XMMRegister vec1, int ae) { |
14626 | 7882 |
ShortBranchVerifier sbv(this); |
7883 |
Label LENGTH_DIFF_LABEL, POP_LABEL, DONE_LABEL, WHILE_HEAD_LABEL; |
|
35113 | 7884 |
Label COMPARE_WIDE_VECTORS_LOOP_FAILED; // used only _LP64 && AVX3 |
33628 | 7885 |
int stride, stride2, adr_stride, adr_stride1, adr_stride2; |
35113 | 7886 |
int stride2x2 = 0x40; |
36061
baa556050d22
8145700: Uninitialised variable in macroAssembler_x86.cpp:7038
thartmann
parents:
35548
diff
changeset
|
7887 |
Address::ScaleFactor scale = Address::no_scale; |
baa556050d22
8145700: Uninitialised variable in macroAssembler_x86.cpp:7038
thartmann
parents:
35548
diff
changeset
|
7888 |
Address::ScaleFactor scale1 = Address::no_scale; |
baa556050d22
8145700: Uninitialised variable in macroAssembler_x86.cpp:7038
thartmann
parents:
35548
diff
changeset
|
7889 |
Address::ScaleFactor scale2 = Address::no_scale; |
33628 | 7890 |
|
35113 | 7891 |
if (ae != StrIntrinsicNode::LL) { |
7892 |
stride2x2 = 0x20; |
|
7893 |
} |
|
7894 |
||
33628 | 7895 |
if (ae == StrIntrinsicNode::LU || ae == StrIntrinsicNode::UL) { |
7896 |
shrl(cnt2, 1); |
|
7897 |
} |
|
14626 | 7898 |
// Compute the minimum of the string lengths and the |
7899 |
// difference of the string lengths (stack). |
|
7900 |
// Do the conditional move stuff |
|
7901 |
movl(result, cnt1); |
|
7902 |
subl(cnt1, cnt2); |
|
7903 |
push(cnt1); |
|
35113 | 7904 |
cmov32(Assembler::lessEqual, cnt2, result); // cnt2 = min(cnt1, cnt2) |
14626 | 7905 |
|
7906 |
// Is the minimum length zero? |
|
7907 |
testl(cnt2, cnt2); |
|
7908 |
jcc(Assembler::zero, LENGTH_DIFF_LABEL); |
|
33628 | 7909 |
if (ae == StrIntrinsicNode::LL) { |
7910 |
// Load first bytes |
|
35113 | 7911 |
load_unsigned_byte(result, Address(str1, 0)); // result = str1[0] |
7912 |
load_unsigned_byte(cnt1, Address(str2, 0)); // cnt1 = str2[0] |
|
33628 | 7913 |
} else if (ae == StrIntrinsicNode::UU) { |
7914 |
// Load first characters |
|
7915 |
load_unsigned_short(result, Address(str1, 0)); |
|
7916 |
load_unsigned_short(cnt1, Address(str2, 0)); |
|
7917 |
} else { |
|
7918 |
load_unsigned_byte(result, Address(str1, 0)); |
|
7919 |
load_unsigned_short(cnt1, Address(str2, 0)); |
|
7920 |
} |
|
14626 | 7921 |
subl(result, cnt1); |
7922 |
jcc(Assembler::notZero, POP_LABEL); |
|
33628 | 7923 |
|
7924 |
if (ae == StrIntrinsicNode::UU) { |
|
7925 |
// Divide length by 2 to get number of chars |
|
7926 |
shrl(cnt2, 1); |
|
7927 |
} |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7928 |
cmpl(cnt2, 1); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7929 |
jcc(Assembler::equal, LENGTH_DIFF_LABEL); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7930 |
|
33628 | 7931 |
// Check if the strings start at the same location and setup scale and stride |
7932 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
|
7933 |
cmpptr(str1, str2); |
|
7934 |
jcc(Assembler::equal, LENGTH_DIFF_LABEL); |
|
7935 |
if (ae == StrIntrinsicNode::LL) { |
|
7936 |
scale = Address::times_1; |
|
7937 |
stride = 16; |
|
7938 |
} else { |
|
7939 |
scale = Address::times_2; |
|
7940 |
stride = 8; |
|
7941 |
} |
|
7942 |
} else { |
|
7943 |
scale1 = Address::times_1; |
|
7944 |
scale2 = Address::times_2; |
|
36061
baa556050d22
8145700: Uninitialised variable in macroAssembler_x86.cpp:7038
thartmann
parents:
35548
diff
changeset
|
7945 |
// scale not used |
33628 | 7946 |
stride = 8; |
7947 |
} |
|
14626 | 7948 |
|
15612
d4073ad8ce3d
8007708: compiler/6855215 assert(VM_Version::supports_sse4_2())
kvn
parents:
15483
diff
changeset
|
7949 |
if (UseAVX >= 2 && UseSSE42Intrinsics) { |
34207
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7950 |
assert(UseSSE >= 4, "SSE4 must be enabled for SSE4.2 intrinsics to be available"); |
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7951 |
Label COMPARE_WIDE_VECTORS, VECTOR_NOT_EQUAL, COMPARE_WIDE_TAIL, COMPARE_SMALL_STR; |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7952 |
Label COMPARE_WIDE_VECTORS_LOOP, COMPARE_16_CHARS, COMPARE_INDEX_CHAR; |
35113 | 7953 |
Label COMPARE_WIDE_VECTORS_LOOP_AVX2; |
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7954 |
Label COMPARE_TAIL_LONG; |
35113 | 7955 |
Label COMPARE_WIDE_VECTORS_LOOP_AVX3; // used only _LP64 && AVX3 |
7956 |
||
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7957 |
int pcmpmask = 0x19; |
33628 | 7958 |
if (ae == StrIntrinsicNode::LL) { |
7959 |
pcmpmask &= ~0x01; |
|
7960 |
} |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7961 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7962 |
// Setup to compare 16-chars (32-bytes) vectors, |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7963 |
// start from first character again because it has aligned address. |
33628 | 7964 |
if (ae == StrIntrinsicNode::LL) { |
7965 |
stride2 = 32; |
|
7966 |
} else { |
|
7967 |
stride2 = 16; |
|
7968 |
} |
|
7969 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
|
7970 |
adr_stride = stride << scale; |
|
7971 |
} else { |
|
7972 |
adr_stride1 = 8; //stride << scale1; |
|
7973 |
adr_stride2 = 16; //stride << scale2; |
|
7974 |
} |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7975 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7976 |
assert(result == rax && cnt2 == rdx && cnt1 == rcx, "pcmpestri"); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7977 |
// rax and rdx are used by pcmpestri as elements counters |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7978 |
movl(result, cnt2); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7979 |
andl(cnt2, ~(stride2-1)); // cnt2 holds the vector count |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7980 |
jcc(Assembler::zero, COMPARE_TAIL_LONG); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7981 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7982 |
// fast path : compare first 2 8-char vectors. |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7983 |
bind(COMPARE_16_CHARS); |
33628 | 7984 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
7985 |
movdqu(vec1, Address(str1, 0)); |
|
7986 |
} else { |
|
7987 |
pmovzxbw(vec1, Address(str1, 0)); |
|
7988 |
} |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7989 |
pcmpestri(vec1, Address(str2, 0), pcmpmask); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7990 |
jccb(Assembler::below, COMPARE_INDEX_CHAR); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7991 |
|
33628 | 7992 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
7993 |
movdqu(vec1, Address(str1, adr_stride)); |
|
7994 |
pcmpestri(vec1, Address(str2, adr_stride), pcmpmask); |
|
7995 |
} else { |
|
7996 |
pmovzxbw(vec1, Address(str1, adr_stride1)); |
|
7997 |
pcmpestri(vec1, Address(str2, adr_stride2), pcmpmask); |
|
7998 |
} |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7999 |
jccb(Assembler::aboveEqual, COMPARE_WIDE_VECTORS); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8000 |
addl(cnt1, stride); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8001 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8002 |
// Compare the characters at index in cnt1 |
33628 | 8003 |
bind(COMPARE_INDEX_CHAR); // cnt1 has the offset of the mismatching character |
8004 |
load_next_elements(result, cnt2, str1, str2, scale, scale1, scale2, cnt1, ae); |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8005 |
subl(result, cnt2); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8006 |
jmp(POP_LABEL); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8007 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8008 |
// Setup the registers to start vector comparison loop |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8009 |
bind(COMPARE_WIDE_VECTORS); |
33628 | 8010 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
8011 |
lea(str1, Address(str1, result, scale)); |
|
8012 |
lea(str2, Address(str2, result, scale)); |
|
8013 |
} else { |
|
8014 |
lea(str1, Address(str1, result, scale1)); |
|
8015 |
lea(str2, Address(str2, result, scale2)); |
|
8016 |
} |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8017 |
subl(result, stride2); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8018 |
subl(cnt2, stride2); |
35113 | 8019 |
jcc(Assembler::zero, COMPARE_WIDE_TAIL); |
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8020 |
negptr(result); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8021 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8022 |
// In a loop, compare 16-chars (32-bytes) at once using (vpxor+vptest) |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8023 |
bind(COMPARE_WIDE_VECTORS_LOOP); |
35113 | 8024 |
|
8025 |
#ifdef _LP64 |
|
8026 |
if (VM_Version::supports_avx512vlbw()) { // trying 64 bytes fast loop |
|
8027 |
cmpl(cnt2, stride2x2); |
|
8028 |
jccb(Assembler::below, COMPARE_WIDE_VECTORS_LOOP_AVX2); |
|
8029 |
testl(cnt2, stride2x2-1); // cnt2 holds the vector count |
|
8030 |
jccb(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP_AVX2); // means we cannot subtract by 0x40 |
|
8031 |
||
8032 |
bind(COMPARE_WIDE_VECTORS_LOOP_AVX3); // the hottest loop |
|
8033 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
|
8034 |
evmovdquq(vec1, Address(str1, result, scale), Assembler::AVX_512bit); |
|
8035 |
evpcmpeqb(k7, vec1, Address(str2, result, scale), Assembler::AVX_512bit); // k7 == 11..11, if operands equal, otherwise k7 has some 0 |
|
8036 |
} else { |
|
8037 |
vpmovzxbw(vec1, Address(str1, result, scale1), Assembler::AVX_512bit); |
|
8038 |
evpcmpeqb(k7, vec1, Address(str2, result, scale2), Assembler::AVX_512bit); // k7 == 11..11, if operands equal, otherwise k7 has some 0 |
|
8039 |
} |
|
8040 |
kortestql(k7, k7); |
|
8041 |
jcc(Assembler::aboveEqual, COMPARE_WIDE_VECTORS_LOOP_FAILED); // miscompare |
|
8042 |
addptr(result, stride2x2); // update since we already compared at this addr |
|
8043 |
subl(cnt2, stride2x2); // and sub the size too |
|
8044 |
jccb(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP_AVX3); |
|
8045 |
||
8046 |
vpxor(vec1, vec1); |
|
8047 |
jmpb(COMPARE_WIDE_TAIL); |
|
8048 |
}//if (VM_Version::supports_avx512vlbw()) |
|
8049 |
#endif // _LP64 |
|
8050 |
||
8051 |
||
8052 |
bind(COMPARE_WIDE_VECTORS_LOOP_AVX2); |
|
33628 | 8053 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
8054 |
vmovdqu(vec1, Address(str1, result, scale)); |
|
8055 |
vpxor(vec1, Address(str2, result, scale)); |
|
8056 |
} else { |
|
34203 | 8057 |
vpmovzxbw(vec1, Address(str1, result, scale1), Assembler::AVX_256bit); |
33628 | 8058 |
vpxor(vec1, Address(str2, result, scale2)); |
8059 |
} |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8060 |
vptest(vec1, vec1); |
35113 | 8061 |
jcc(Assembler::notZero, VECTOR_NOT_EQUAL); |
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8062 |
addptr(result, stride2); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8063 |
subl(cnt2, stride2); |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
8064 |
jcc(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP); |
16624
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
8065 |
// clean upper bits of YMM registers |
30299 | 8066 |
vpxor(vec1, vec1); |
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8067 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8068 |
// compare wide vectors tail |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8069 |
bind(COMPARE_WIDE_TAIL); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8070 |
testptr(result, result); |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
8071 |
jcc(Assembler::zero, LENGTH_DIFF_LABEL); |
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8072 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8073 |
movl(result, stride2); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8074 |
movl(cnt2, result); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8075 |
negptr(result); |
35113 | 8076 |
jmp(COMPARE_WIDE_VECTORS_LOOP_AVX2); |
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8077 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8078 |
// Identifies the mismatching (higher or lower)16-bytes in the 32-byte vectors. |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8079 |
bind(VECTOR_NOT_EQUAL); |
16624
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
8080 |
// clean upper bits of YMM registers |
30299 | 8081 |
vpxor(vec1, vec1); |
33628 | 8082 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
8083 |
lea(str1, Address(str1, result, scale)); |
|
8084 |
lea(str2, Address(str2, result, scale)); |
|
8085 |
} else { |
|
8086 |
lea(str1, Address(str1, result, scale1)); |
|
8087 |
lea(str2, Address(str2, result, scale2)); |
|
8088 |
} |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8089 |
jmp(COMPARE_16_CHARS); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8090 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8091 |
// Compare tail chars, length between 1 to 15 chars |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8092 |
bind(COMPARE_TAIL_LONG); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8093 |
movl(cnt2, result); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8094 |
cmpl(cnt2, stride); |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
8095 |
jcc(Assembler::less, COMPARE_SMALL_STR); |
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8096 |
|
33628 | 8097 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
8098 |
movdqu(vec1, Address(str1, 0)); |
|
8099 |
} else { |
|
8100 |
pmovzxbw(vec1, Address(str1, 0)); |
|
8101 |
} |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8102 |
pcmpestri(vec1, Address(str2, 0), pcmpmask); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8103 |
jcc(Assembler::below, COMPARE_INDEX_CHAR); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8104 |
subptr(cnt2, stride); |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
8105 |
jcc(Assembler::zero, LENGTH_DIFF_LABEL); |
33628 | 8106 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
8107 |
lea(str1, Address(str1, result, scale)); |
|
8108 |
lea(str2, Address(str2, result, scale)); |
|
8109 |
} else { |
|
8110 |
lea(str1, Address(str1, result, scale1)); |
|
8111 |
lea(str2, Address(str2, result, scale2)); |
|
8112 |
} |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8113 |
negptr(cnt2); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8114 |
jmpb(WHILE_HEAD_LABEL); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8115 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8116 |
bind(COMPARE_SMALL_STR); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8117 |
} else if (UseSSE42Intrinsics) { |
34207
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
8118 |
assert(UseSSE >= 4, "SSE4 must be enabled for SSE4.2 intrinsics to be available"); |
14626 | 8119 |
Label COMPARE_WIDE_VECTORS, VECTOR_NOT_EQUAL, COMPARE_TAIL; |
8120 |
int pcmpmask = 0x19; |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8121 |
// Setup to compare 8-char (16-byte) vectors, |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8122 |
// start from first character again because it has aligned address. |
14626 | 8123 |
movl(result, cnt2); |
8124 |
andl(cnt2, ~(stride - 1)); // cnt2 holds the vector count |
|
33628 | 8125 |
if (ae == StrIntrinsicNode::LL) { |
8126 |
pcmpmask &= ~0x01; |
|
8127 |
} |
|
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
8128 |
jcc(Assembler::zero, COMPARE_TAIL); |
33628 | 8129 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
8130 |
lea(str1, Address(str1, result, scale)); |
|
8131 |
lea(str2, Address(str2, result, scale)); |
|
8132 |
} else { |
|
8133 |
lea(str1, Address(str1, result, scale1)); |
|
8134 |
lea(str2, Address(str2, result, scale2)); |
|
8135 |
} |
|
14626 | 8136 |
negptr(result); |
8137 |
||
8138 |
// pcmpestri |
|
8139 |
// inputs: |
|
8140 |
// vec1- substring |
|
8141 |
// rax - negative string length (elements count) |
|
28719 | 8142 |
// mem - scanned string |
14626 | 8143 |
// rdx - string length (elements count) |
8144 |
// pcmpmask - cmp mode: 11000 (string compare with negated result) |
|
8145 |
// + 00 (unsigned bytes) or + 01 (unsigned shorts) |
|
8146 |
// outputs: |
|
8147 |
// rcx - first mismatched element index |
|
8148 |
assert(result == rax && cnt2 == rdx && cnt1 == rcx, "pcmpestri"); |
|
8149 |
||
8150 |
bind(COMPARE_WIDE_VECTORS); |
|
33628 | 8151 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
8152 |
movdqu(vec1, Address(str1, result, scale)); |
|
8153 |
pcmpestri(vec1, Address(str2, result, scale), pcmpmask); |
|
8154 |
} else { |
|
8155 |
pmovzxbw(vec1, Address(str1, result, scale1)); |
|
8156 |
pcmpestri(vec1, Address(str2, result, scale2), pcmpmask); |
|
8157 |
} |
|
14626 | 8158 |
// After pcmpestri cnt1(rcx) contains mismatched element index |
8159 |
||
8160 |
jccb(Assembler::below, VECTOR_NOT_EQUAL); // CF==1 |
|
8161 |
addptr(result, stride); |
|
8162 |
subptr(cnt2, stride); |
|
8163 |
jccb(Assembler::notZero, COMPARE_WIDE_VECTORS); |
|
8164 |
||
8165 |
// compare wide vectors tail |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8166 |
testptr(result, result); |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
8167 |
jcc(Assembler::zero, LENGTH_DIFF_LABEL); |
14626 | 8168 |
|
8169 |
movl(cnt2, stride); |
|
8170 |
movl(result, stride); |
|
8171 |
negptr(result); |
|
33628 | 8172 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
8173 |
movdqu(vec1, Address(str1, result, scale)); |
|
8174 |
pcmpestri(vec1, Address(str2, result, scale), pcmpmask); |
|
8175 |
} else { |
|
8176 |
pmovzxbw(vec1, Address(str1, result, scale1)); |
|
8177 |
pcmpestri(vec1, Address(str2, result, scale2), pcmpmask); |
|
8178 |
} |
|
14626 | 8179 |
jccb(Assembler::aboveEqual, LENGTH_DIFF_LABEL); |
8180 |
||
8181 |
// Mismatched characters in the vectors |
|
8182 |
bind(VECTOR_NOT_EQUAL); |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8183 |
addptr(cnt1, result); |
33628 | 8184 |
load_next_elements(result, cnt2, str1, str2, scale, scale1, scale2, cnt1, ae); |
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8185 |
subl(result, cnt2); |
14626 | 8186 |
jmpb(POP_LABEL); |
8187 |
||
8188 |
bind(COMPARE_TAIL); // limit is zero |
|
8189 |
movl(cnt2, result); |
|
8190 |
// Fallthru to tail compare |
|
8191 |
} |
|
8192 |
// Shift str2 and str1 to the end of the arrays, negate min |
|
33628 | 8193 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
8194 |
lea(str1, Address(str1, cnt2, scale)); |
|
8195 |
lea(str2, Address(str2, cnt2, scale)); |
|
8196 |
} else { |
|
8197 |
lea(str1, Address(str1, cnt2, scale1)); |
|
8198 |
lea(str2, Address(str2, cnt2, scale2)); |
|
8199 |
} |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8200 |
decrementl(cnt2); // first character was compared already |
14626 | 8201 |
negptr(cnt2); |
8202 |
||
8203 |
// Compare the rest of the elements |
|
8204 |
bind(WHILE_HEAD_LABEL); |
|
33628 | 8205 |
load_next_elements(result, cnt1, str1, str2, scale, scale1, scale2, cnt2, ae); |
14626 | 8206 |
subl(result, cnt1); |
8207 |
jccb(Assembler::notZero, POP_LABEL); |
|
8208 |
increment(cnt2); |
|
8209 |
jccb(Assembler::notZero, WHILE_HEAD_LABEL); |
|
8210 |
||
8211 |
// Strings are equal up to min length. Return the length difference. |
|
8212 |
bind(LENGTH_DIFF_LABEL); |
|
8213 |
pop(result); |
|
33628 | 8214 |
if (ae == StrIntrinsicNode::UU) { |
8215 |
// Divide diff by 2 to get number of chars |
|
8216 |
sarl(result, 1); |
|
8217 |
} |
|
14626 | 8218 |
jmpb(DONE_LABEL); |
8219 |
||
35113 | 8220 |
#ifdef _LP64 |
8221 |
if (VM_Version::supports_avx512vlbw()) { |
|
8222 |
||
8223 |
bind(COMPARE_WIDE_VECTORS_LOOP_FAILED); |
|
8224 |
||
8225 |
kmovql(cnt1, k7); |
|
8226 |
notq(cnt1); |
|
8227 |
bsfq(cnt2, cnt1); |
|
8228 |
if (ae != StrIntrinsicNode::LL) { |
|
8229 |
// Divide diff by 2 to get number of chars |
|
8230 |
sarl(cnt2, 1); |
|
8231 |
} |
|
8232 |
addq(result, cnt2); |
|
8233 |
if (ae == StrIntrinsicNode::LL) { |
|
8234 |
load_unsigned_byte(cnt1, Address(str2, result)); |
|
8235 |
load_unsigned_byte(result, Address(str1, result)); |
|
8236 |
} else if (ae == StrIntrinsicNode::UU) { |
|
8237 |
load_unsigned_short(cnt1, Address(str2, result, scale)); |
|
8238 |
load_unsigned_short(result, Address(str1, result, scale)); |
|
8239 |
} else { |
|
8240 |
load_unsigned_short(cnt1, Address(str2, result, scale2)); |
|
8241 |
load_unsigned_byte(result, Address(str1, result, scale1)); |
|
8242 |
} |
|
8243 |
subl(result, cnt1); |
|
8244 |
jmpb(POP_LABEL); |
|
8245 |
}//if (VM_Version::supports_avx512vlbw()) |
|
8246 |
#endif // _LP64 |
|
8247 |
||
14626 | 8248 |
// Discard the stored length difference |
8249 |
bind(POP_LABEL); |
|
8250 |
pop(cnt1); |
|
8251 |
||
8252 |
// That's it |
|
8253 |
bind(DONE_LABEL); |
|
33628 | 8254 |
if(ae == StrIntrinsicNode::UL) { |
8255 |
negl(result); |
|
8256 |
} |
|
35113 | 8257 |
|
33628 | 8258 |
} |
8259 |
||
8260 |
// Search for Non-ASCII character (Negative byte value) in a byte array, |
|
8261 |
// return true if it has any and false otherwise. |
|
8262 |
void MacroAssembler::has_negatives(Register ary1, Register len, |
|
8263 |
Register result, Register tmp1, |
|
8264 |
XMMRegister vec1, XMMRegister vec2) { |
|
8265 |
||
8266 |
// rsi: byte array |
|
8267 |
// rcx: len |
|
8268 |
// rax: result |
|
14626 | 8269 |
ShortBranchVerifier sbv(this); |
33628 | 8270 |
assert_different_registers(ary1, len, result, tmp1); |
8271 |
assert_different_registers(vec1, vec2); |
|
8272 |
Label TRUE_LABEL, FALSE_LABEL, DONE, COMPARE_CHAR, COMPARE_VECTORS, COMPARE_BYTE; |
|
8273 |
||
8274 |
// len == 0 |
|
8275 |
testl(len, len); |
|
8276 |
jcc(Assembler::zero, FALSE_LABEL); |
|
8277 |
||
8278 |
movl(result, len); // copy |
|
8279 |
||
34207
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
8280 |
if (UseAVX >= 2 && UseSSE >= 2) { |
33628 | 8281 |
// With AVX2, use 32-byte vector compare |
8282 |
Label COMPARE_WIDE_VECTORS, COMPARE_TAIL; |
|
8283 |
||
8284 |
// Compare 32-byte vectors |
|
8285 |
andl(result, 0x0000001f); // tail count (in bytes) |
|
8286 |
andl(len, 0xffffffe0); // vector count (in bytes) |
|
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
8287 |
jcc(Assembler::zero, COMPARE_TAIL); |
33628 | 8288 |
|
8289 |
lea(ary1, Address(ary1, len, Address::times_1)); |
|
8290 |
negptr(len); |
|
8291 |
||
8292 |
movl(tmp1, 0x80808080); // create mask to test for Unicode chars in vector |
|
8293 |
movdl(vec2, tmp1); |
|
8294 |
vpbroadcastd(vec2, vec2); |
|
8295 |
||
8296 |
bind(COMPARE_WIDE_VECTORS); |
|
8297 |
vmovdqu(vec1, Address(ary1, len, Address::times_1)); |
|
8298 |
vptest(vec1, vec2); |
|
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
8299 |
jcc(Assembler::notZero, TRUE_LABEL); |
33628 | 8300 |
addptr(len, 32); |
8301 |
jcc(Assembler::notZero, COMPARE_WIDE_VECTORS); |
|
8302 |
||
8303 |
testl(result, result); |
|
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
8304 |
jcc(Assembler::zero, FALSE_LABEL); |
33628 | 8305 |
|
8306 |
vmovdqu(vec1, Address(ary1, result, Address::times_1, -32)); |
|
8307 |
vptest(vec1, vec2); |
|
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
8308 |
jcc(Assembler::notZero, TRUE_LABEL); |
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
8309 |
jmp(FALSE_LABEL); |
33628 | 8310 |
|
8311 |
bind(COMPARE_TAIL); // len is zero |
|
8312 |
movl(len, result); |
|
8313 |
// Fallthru to tail compare |
|
8314 |
} else if (UseSSE42Intrinsics) { |
|
34207
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
8315 |
assert(UseSSE >= 4, "SSE4 must be for SSE4.2 intrinsics to be available"); |
33628 | 8316 |
// With SSE4.2, use double quad vector compare |
8317 |
Label COMPARE_WIDE_VECTORS, COMPARE_TAIL; |
|
8318 |
||
8319 |
// Compare 16-byte vectors |
|
8320 |
andl(result, 0x0000000f); // tail count (in bytes) |
|
8321 |
andl(len, 0xfffffff0); // vector count (in bytes) |
|
8322 |
jccb(Assembler::zero, COMPARE_TAIL); |
|
8323 |
||
8324 |
lea(ary1, Address(ary1, len, Address::times_1)); |
|
8325 |
negptr(len); |
|
8326 |
||
8327 |
movl(tmp1, 0x80808080); |
|
8328 |
movdl(vec2, tmp1); |
|
8329 |
pshufd(vec2, vec2, 0); |
|
8330 |
||
8331 |
bind(COMPARE_WIDE_VECTORS); |
|
8332 |
movdqu(vec1, Address(ary1, len, Address::times_1)); |
|
8333 |
ptest(vec1, vec2); |
|
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
8334 |
jcc(Assembler::notZero, TRUE_LABEL); |
33628 | 8335 |
addptr(len, 16); |
8336 |
jcc(Assembler::notZero, COMPARE_WIDE_VECTORS); |
|
8337 |
||
8338 |
testl(result, result); |
|
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
8339 |
jcc(Assembler::zero, FALSE_LABEL); |
33628 | 8340 |
|
8341 |
movdqu(vec1, Address(ary1, result, Address::times_1, -16)); |
|
8342 |
ptest(vec1, vec2); |
|
8343 |
jccb(Assembler::notZero, TRUE_LABEL); |
|
8344 |
jmpb(FALSE_LABEL); |
|
8345 |
||
8346 |
bind(COMPARE_TAIL); // len is zero |
|
8347 |
movl(len, result); |
|
8348 |
// Fallthru to tail compare |
|
8349 |
} |
|
8350 |
||
8351 |
// Compare 4-byte vectors |
|
8352 |
andl(len, 0xfffffffc); // vector count (in bytes) |
|
8353 |
jccb(Assembler::zero, COMPARE_CHAR); |
|
8354 |
||
8355 |
lea(ary1, Address(ary1, len, Address::times_1)); |
|
8356 |
negptr(len); |
|
8357 |
||
8358 |
bind(COMPARE_VECTORS); |
|
8359 |
movl(tmp1, Address(ary1, len, Address::times_1)); |
|
8360 |
andl(tmp1, 0x80808080); |
|
8361 |
jccb(Assembler::notZero, TRUE_LABEL); |
|
8362 |
addptr(len, 4); |
|
8363 |
jcc(Assembler::notZero, COMPARE_VECTORS); |
|
8364 |
||
8365 |
// Compare trailing char (final 2 bytes), if any |
|
8366 |
bind(COMPARE_CHAR); |
|
8367 |
testl(result, 0x2); // tail char |
|
8368 |
jccb(Assembler::zero, COMPARE_BYTE); |
|
8369 |
load_unsigned_short(tmp1, Address(ary1, 0)); |
|
8370 |
andl(tmp1, 0x00008080); |
|
8371 |
jccb(Assembler::notZero, TRUE_LABEL); |
|
8372 |
subptr(result, 2); |
|
8373 |
lea(ary1, Address(ary1, 2)); |
|
8374 |
||
8375 |
bind(COMPARE_BYTE); |
|
8376 |
testl(result, 0x1); // tail byte |
|
8377 |
jccb(Assembler::zero, FALSE_LABEL); |
|
8378 |
load_unsigned_byte(tmp1, Address(ary1, 0)); |
|
8379 |
andl(tmp1, 0x00000080); |
|
8380 |
jccb(Assembler::notEqual, TRUE_LABEL); |
|
8381 |
jmpb(FALSE_LABEL); |
|
8382 |
||
8383 |
bind(TRUE_LABEL); |
|
8384 |
movl(result, 1); // return true |
|
8385 |
jmpb(DONE); |
|
8386 |
||
8387 |
bind(FALSE_LABEL); |
|
8388 |
xorl(result, result); // return false |
|
8389 |
||
8390 |
// That's it |
|
8391 |
bind(DONE); |
|
34207
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
8392 |
if (UseAVX >= 2 && UseSSE >= 2) { |
33628 | 8393 |
// clean upper bits of YMM registers |
8394 |
vpxor(vec1, vec1); |
|
8395 |
vpxor(vec2, vec2); |
|
8396 |
} |
|
8397 |
} |
|
8398 |
||
8399 |
// Compare char[] or byte[] arrays aligned to 4 bytes or substrings. |
|
8400 |
void MacroAssembler::arrays_equals(bool is_array_equ, Register ary1, Register ary2, |
|
8401 |
Register limit, Register result, Register chr, |
|
8402 |
XMMRegister vec1, XMMRegister vec2, bool is_char) { |
|
8403 |
ShortBranchVerifier sbv(this); |
|
8404 |
Label TRUE_LABEL, FALSE_LABEL, DONE, COMPARE_VECTORS, COMPARE_CHAR, COMPARE_BYTE; |
|
14626 | 8405 |
|
8406 |
int length_offset = arrayOopDesc::length_offset_in_bytes(); |
|
33628 | 8407 |
int base_offset = arrayOopDesc::base_offset_in_bytes(is_char ? T_CHAR : T_BYTE); |
14626 | 8408 |
|
8409 |
if (is_array_equ) { |
|
33628 | 8410 |
// Check the input args |
8411 |
cmpptr(ary1, ary2); |
|
8412 |
jcc(Assembler::equal, TRUE_LABEL); |
|
8413 |
||
14626 | 8414 |
// Need additional checks for arrays_equals. |
8415 |
testptr(ary1, ary1); |
|
8416 |
jcc(Assembler::zero, FALSE_LABEL); |
|
8417 |
testptr(ary2, ary2); |
|
8418 |
jcc(Assembler::zero, FALSE_LABEL); |
|
8419 |
||
8420 |
// Check the lengths |
|
8421 |
movl(limit, Address(ary1, length_offset)); |
|
8422 |
cmpl(limit, Address(ary2, length_offset)); |
|
8423 |
jcc(Assembler::notEqual, FALSE_LABEL); |
|
8424 |
} |
|
8425 |
||
8426 |
// count == 0 |
|
8427 |
testl(limit, limit); |
|
8428 |
jcc(Assembler::zero, TRUE_LABEL); |
|
8429 |
||
8430 |
if (is_array_equ) { |
|
8431 |
// Load array address |
|
8432 |
lea(ary1, Address(ary1, base_offset)); |
|
8433 |
lea(ary2, Address(ary2, base_offset)); |
|
8434 |
} |
|
8435 |
||
33628 | 8436 |
if (is_array_equ && is_char) { |
8437 |
// arrays_equals when used for char[]. |
|
8438 |
shll(limit, 1); // byte count != 0 |
|
8439 |
} |
|
14626 | 8440 |
movl(result, limit); // copy |
8441 |
||
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8442 |
if (UseAVX >= 2) { |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8443 |
// With AVX2, use 32-byte vector compare |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8444 |
Label COMPARE_WIDE_VECTORS, COMPARE_TAIL; |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8445 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8446 |
// Compare 32-byte vectors |
33628 | 8447 |
andl(result, 0x0000001f); // tail count (in bytes) |
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8448 |
andl(limit, 0xffffffe0); // vector count (in bytes) |
35136
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8449 |
jcc(Assembler::zero, COMPARE_TAIL); |
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8450 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8451 |
lea(ary1, Address(ary1, limit, Address::times_1)); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8452 |
lea(ary2, Address(ary2, limit, Address::times_1)); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8453 |
negptr(limit); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8454 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8455 |
bind(COMPARE_WIDE_VECTORS); |
35136
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8456 |
|
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8457 |
#ifdef _LP64 |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8458 |
if (VM_Version::supports_avx512vlbw()) { // trying 64 bytes fast loop |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8459 |
Label COMPARE_WIDE_VECTORS_LOOP_AVX2, COMPARE_WIDE_VECTORS_LOOP_AVX3; |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8460 |
|
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8461 |
cmpl(limit, -64); |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8462 |
jccb(Assembler::greater, COMPARE_WIDE_VECTORS_LOOP_AVX2); |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8463 |
|
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8464 |
bind(COMPARE_WIDE_VECTORS_LOOP_AVX3); // the hottest loop |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8465 |
|
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8466 |
evmovdquq(vec1, Address(ary1, limit, Address::times_1), Assembler::AVX_512bit); |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8467 |
evpcmpeqb(k7, vec1, Address(ary2, limit, Address::times_1), Assembler::AVX_512bit); |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8468 |
kortestql(k7, k7); |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8469 |
jcc(Assembler::aboveEqual, FALSE_LABEL); // miscompare |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8470 |
addptr(limit, 64); // update since we already compared at this addr |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8471 |
cmpl(limit, -64); |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8472 |
jccb(Assembler::lessEqual, COMPARE_WIDE_VECTORS_LOOP_AVX3); |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8473 |
|
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8474 |
// At this point we may still need to compare -limit+result bytes. |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8475 |
// We could execute the next two instruction and just continue via non-wide path: |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8476 |
// cmpl(limit, 0); |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8477 |
// jcc(Assembler::equal, COMPARE_TAIL); // true |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8478 |
// But since we stopped at the points ary{1,2}+limit which are |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8479 |
// not farther than 64 bytes from the ends of arrays ary{1,2}+result |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8480 |
// (|limit| <= 32 and result < 32), |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8481 |
// we may just compare the last 64 bytes. |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8482 |
// |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8483 |
addptr(result, -64); // it is safe, bc we just came from this area |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8484 |
evmovdquq(vec1, Address(ary1, result, Address::times_1), Assembler::AVX_512bit); |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8485 |
evpcmpeqb(k7, vec1, Address(ary2, result, Address::times_1), Assembler::AVX_512bit); |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8486 |
kortestql(k7, k7); |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8487 |
jcc(Assembler::aboveEqual, FALSE_LABEL); // miscompare |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8488 |
|
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8489 |
jmp(TRUE_LABEL); |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8490 |
|
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8491 |
bind(COMPARE_WIDE_VECTORS_LOOP_AVX2); |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8492 |
|
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8493 |
}//if (VM_Version::supports_avx512vlbw()) |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8494 |
#endif //_LP64 |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
8495 |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8496 |
vmovdqu(vec1, Address(ary1, limit, Address::times_1)); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8497 |
vmovdqu(vec2, Address(ary2, limit, Address::times_1)); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8498 |
vpxor(vec1, vec2); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8499 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8500 |
vptest(vec1, vec1); |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
8501 |
jcc(Assembler::notZero, FALSE_LABEL); |
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8502 |
addptr(limit, 32); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8503 |
jcc(Assembler::notZero, COMPARE_WIDE_VECTORS); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8504 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8505 |
testl(result, result); |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
8506 |
jcc(Assembler::zero, TRUE_LABEL); |
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8507 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8508 |
vmovdqu(vec1, Address(ary1, result, Address::times_1, -32)); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8509 |
vmovdqu(vec2, Address(ary2, result, Address::times_1, -32)); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8510 |
vpxor(vec1, vec2); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8511 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8512 |
vptest(vec1, vec1); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8513 |
jccb(Assembler::notZero, FALSE_LABEL); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8514 |
jmpb(TRUE_LABEL); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8515 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8516 |
bind(COMPARE_TAIL); // limit is zero |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8517 |
movl(limit, result); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8518 |
// Fallthru to tail compare |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
8519 |
} else if (UseSSE42Intrinsics) { |
34207
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
8520 |
assert(UseSSE >= 4, "SSE4 must be enabled for SSE4.2 intrinsics to be available"); |
14626 | 8521 |
// With SSE4.2, use double quad vector compare |
8522 |
Label COMPARE_WIDE_VECTORS, COMPARE_TAIL; |
|
8523 |
||
8524 |
// Compare 16-byte vectors |
|
33628 | 8525 |
andl(result, 0x0000000f); // tail count (in bytes) |
14626 | 8526 |
andl(limit, 0xfffffff0); // vector count (in bytes) |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
8527 |
jcc(Assembler::zero, COMPARE_TAIL); |
14626 | 8528 |
|
8529 |
lea(ary1, Address(ary1, limit, Address::times_1)); |
|
8530 |
lea(ary2, Address(ary2, limit, Address::times_1)); |
|
8531 |
negptr(limit); |
|
8532 |
||
8533 |
bind(COMPARE_WIDE_VECTORS); |
|
8534 |
movdqu(vec1, Address(ary1, limit, Address::times_1)); |
|
8535 |
movdqu(vec2, Address(ary2, limit, Address::times_1)); |
|
8536 |
pxor(vec1, vec2); |
|
8537 |
||
8538 |
ptest(vec1, vec1); |
|
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
8539 |
jcc(Assembler::notZero, FALSE_LABEL); |
14626 | 8540 |
addptr(limit, 16); |
8541 |
jcc(Assembler::notZero, COMPARE_WIDE_VECTORS); |
|
8542 |
||
8543 |
testl(result, result); |
|
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
8544 |
jcc(Assembler::zero, TRUE_LABEL); |
14626 | 8545 |
|
8546 |
movdqu(vec1, Address(ary1, result, Address::times_1, -16)); |
|
8547 |
movdqu(vec2, Address(ary2, result, Address::times_1, -16)); |
|
8548 |
pxor(vec1, vec2); |
|
8549 |
||
8550 |
ptest(vec1, vec1); |
|
8551 |
jccb(Assembler::notZero, FALSE_LABEL); |
|
8552 |
jmpb(TRUE_LABEL); |
|
8553 |
||
8554 |
bind(COMPARE_TAIL); // limit is zero |
|
8555 |
movl(limit, result); |
|
8556 |
// Fallthru to tail compare |
|
8557 |
} |
|
8558 |
||
8559 |
// Compare 4-byte vectors |
|
8560 |
andl(limit, 0xfffffffc); // vector count (in bytes) |
|
8561 |
jccb(Assembler::zero, COMPARE_CHAR); |
|
8562 |
||
8563 |
lea(ary1, Address(ary1, limit, Address::times_1)); |
|
8564 |
lea(ary2, Address(ary2, limit, Address::times_1)); |
|
8565 |
negptr(limit); |
|
8566 |
||
8567 |
bind(COMPARE_VECTORS); |
|
8568 |
movl(chr, Address(ary1, limit, Address::times_1)); |
|
8569 |
cmpl(chr, Address(ary2, limit, Address::times_1)); |
|
8570 |
jccb(Assembler::notEqual, FALSE_LABEL); |
|
8571 |
addptr(limit, 4); |
|
8572 |
jcc(Assembler::notZero, COMPARE_VECTORS); |
|
8573 |
||
8574 |
// Compare trailing char (final 2 bytes), if any |
|
8575 |
bind(COMPARE_CHAR); |
|
8576 |
testl(result, 0x2); // tail char |
|
33628 | 8577 |
jccb(Assembler::zero, COMPARE_BYTE); |
14626 | 8578 |
load_unsigned_short(chr, Address(ary1, 0)); |
8579 |
load_unsigned_short(limit, Address(ary2, 0)); |
|
8580 |
cmpl(chr, limit); |
|
8581 |
jccb(Assembler::notEqual, FALSE_LABEL); |
|
8582 |
||
33628 | 8583 |
if (is_array_equ && is_char) { |
8584 |
bind(COMPARE_BYTE); |
|
8585 |
} else { |
|
8586 |
lea(ary1, Address(ary1, 2)); |
|
8587 |
lea(ary2, Address(ary2, 2)); |
|
8588 |
||
8589 |
bind(COMPARE_BYTE); |
|
8590 |
testl(result, 0x1); // tail byte |
|
8591 |
jccb(Assembler::zero, TRUE_LABEL); |
|
8592 |
load_unsigned_byte(chr, Address(ary1, 0)); |
|
8593 |
load_unsigned_byte(limit, Address(ary2, 0)); |
|
8594 |
cmpl(chr, limit); |
|
8595 |
jccb(Assembler::notEqual, FALSE_LABEL); |
|
8596 |
} |
|
14626 | 8597 |
bind(TRUE_LABEL); |
8598 |
movl(result, 1); // return true |
|
8599 |
jmpb(DONE); |
|
8600 |
||
8601 |
bind(FALSE_LABEL); |
|
8602 |
xorl(result, result); // return false |
|
8603 |
||
8604 |
// That's it |
|
8605 |
bind(DONE); |
|
16624
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
8606 |
if (UseAVX >= 2) { |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
8607 |
// clean upper bits of YMM registers |
30299 | 8608 |
vpxor(vec1, vec1); |
8609 |
vpxor(vec2, vec2); |
|
16624
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
8610 |
} |
14626 | 8611 |
} |
8612 |
||
33628 | 8613 |
#endif |
8614 |
||
14626 | 8615 |
void MacroAssembler::generate_fill(BasicType t, bool aligned, |
8616 |
Register to, Register value, Register count, |
|
8617 |
Register rtmp, XMMRegister xtmp) { |
|
8618 |
ShortBranchVerifier sbv(this); |
|
8619 |
assert_different_registers(to, value, count, rtmp); |
|
8620 |
Label L_exit, L_skip_align1, L_skip_align2, L_fill_byte; |
|
8621 |
Label L_fill_2_bytes, L_fill_4_bytes; |
|
8622 |
||
8623 |
int shift = -1; |
|
8624 |
switch (t) { |
|
8625 |
case T_BYTE: |
|
8626 |
shift = 2; |
|
8627 |
break; |
|
8628 |
case T_SHORT: |
|
8629 |
shift = 1; |
|
8630 |
break; |
|
8631 |
case T_INT: |
|
8632 |
shift = 0; |
|
8633 |
break; |
|
8634 |
default: ShouldNotReachHere(); |
|
8635 |
} |
|
8636 |
||
8637 |
if (t == T_BYTE) { |
|
8638 |
andl(value, 0xff); |
|
8639 |
movl(rtmp, value); |
|
8640 |
shll(rtmp, 8); |
|
8641 |
orl(value, rtmp); |
|
8642 |
} |
|
8643 |
if (t == T_SHORT) { |
|
8644 |
andl(value, 0xffff); |
|
8645 |
} |
|
8646 |
if (t == T_BYTE || t == T_SHORT) { |
|
8647 |
movl(rtmp, value); |
|
8648 |
shll(rtmp, 16); |
|
8649 |
orl(value, rtmp); |
|
8650 |
} |
|
8651 |
||
8652 |
cmpl(count, 2<<shift); // Short arrays (< 8 bytes) fill by element |
|
8653 |
jcc(Assembler::below, L_fill_4_bytes); // use unsigned cmp |
|
8654 |
if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) { |
|
8655 |
// align source address at 4 bytes address boundary |
|
8656 |
if (t == T_BYTE) { |
|
8657 |
// One byte misalignment happens only for byte arrays |
|
8658 |
testptr(to, 1); |
|
8659 |
jccb(Assembler::zero, L_skip_align1); |
|
8660 |
movb(Address(to, 0), value); |
|
8661 |
increment(to); |
|
8662 |
decrement(count); |
|
8663 |
BIND(L_skip_align1); |
|
8664 |
} |
|
8665 |
// Two bytes misalignment happens only for byte and short (char) arrays |
|
8666 |
testptr(to, 2); |
|
8667 |
jccb(Assembler::zero, L_skip_align2); |
|
8668 |
movw(Address(to, 0), value); |
|
8669 |
addptr(to, 2); |
|
8670 |
subl(count, 1<<(shift-1)); |
|
8671 |
BIND(L_skip_align2); |
|
8672 |
} |
|
8673 |
if (UseSSE < 2) { |
|
8674 |
Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes; |
|
8675 |
// Fill 32-byte chunks |
|
8676 |
subl(count, 8 << shift); |
|
8677 |
jcc(Assembler::less, L_check_fill_8_bytes); |
|
8678 |
align(16); |
|
8679 |
||
8680 |
BIND(L_fill_32_bytes_loop); |
|
8681 |
||
8682 |
for (int i = 0; i < 32; i += 4) { |
|
8683 |
movl(Address(to, i), value); |
|
8684 |
} |
|
8685 |
||
8686 |
addptr(to, 32); |
|
8687 |
subl(count, 8 << shift); |
|
8688 |
jcc(Assembler::greaterEqual, L_fill_32_bytes_loop); |
|
8689 |
BIND(L_check_fill_8_bytes); |
|
8690 |
addl(count, 8 << shift); |
|
8691 |
jccb(Assembler::zero, L_exit); |
|
8692 |
jmpb(L_fill_8_bytes); |
|
8693 |
||
8694 |
// |
|
8695 |
// length is too short, just fill qwords |
|
8696 |
// |
|
8697 |
BIND(L_fill_8_bytes_loop); |
|
8698 |
movl(Address(to, 0), value); |
|
8699 |
movl(Address(to, 4), value); |
|
8700 |
addptr(to, 8); |
|
8701 |
BIND(L_fill_8_bytes); |
|
8702 |
subl(count, 1 << (shift + 1)); |
|
8703 |
jcc(Assembler::greaterEqual, L_fill_8_bytes_loop); |
|
8704 |
// fall through to fill 4 bytes |
|
8705 |
} else { |
|
8706 |
Label L_fill_32_bytes; |
|
8707 |
if (!UseUnalignedLoadStores) { |
|
8708 |
// align to 8 bytes, we know we are 4 byte aligned to start |
|
8709 |
testptr(to, 4); |
|
8710 |
jccb(Assembler::zero, L_fill_32_bytes); |
|
8711 |
movl(Address(to, 0), value); |
|
8712 |
addptr(to, 4); |
|
8713 |
subl(count, 1<<shift); |
|
8714 |
} |
|
8715 |
BIND(L_fill_32_bytes); |
|
8716 |
{ |
|
8717 |
assert( UseSSE >= 2, "supported cpu only" ); |
|
8718 |
Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes; |
|
30624 | 8719 |
if (UseAVX > 2) { |
8720 |
movl(rtmp, 0xffff); |
|
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
8721 |
kmovwl(k1, rtmp); |
30624 | 8722 |
} |
14626 | 8723 |
movdl(xtmp, value); |
30624 | 8724 |
if (UseAVX > 2 && UseUnalignedLoadStores) { |
8725 |
// Fill 64-byte chunks |
|
8726 |
Label L_fill_64_bytes_loop, L_check_fill_32_bytes; |
|
8727 |
evpbroadcastd(xtmp, xtmp, Assembler::AVX_512bit); |
|
8728 |
||
8729 |
subl(count, 16 << shift); |
|
8730 |
jcc(Assembler::less, L_check_fill_32_bytes); |
|
8731 |
align(16); |
|
8732 |
||
8733 |
BIND(L_fill_64_bytes_loop); |
|
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
8734 |
evmovdqul(Address(to, 0), xtmp, Assembler::AVX_512bit); |
30624 | 8735 |
addptr(to, 64); |
8736 |
subl(count, 16 << shift); |
|
8737 |
jcc(Assembler::greaterEqual, L_fill_64_bytes_loop); |
|
8738 |
||
8739 |
BIND(L_check_fill_32_bytes); |
|
8740 |
addl(count, 8 << shift); |
|
8741 |
jccb(Assembler::less, L_check_fill_8_bytes); |
|
34162 | 8742 |
vmovdqu(Address(to, 0), xtmp); |
30624 | 8743 |
addptr(to, 32); |
8744 |
subl(count, 8 << shift); |
|
8745 |
||
8746 |
BIND(L_check_fill_8_bytes); |
|
8747 |
} else if (UseAVX == 2 && UseUnalignedLoadStores) { |
|
15115
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8748 |
// Fill 64-byte chunks |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8749 |
Label L_fill_64_bytes_loop, L_check_fill_32_bytes; |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8750 |
vpbroadcastd(xtmp, xtmp); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8751 |
|
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8752 |
subl(count, 16 << shift); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8753 |
jcc(Assembler::less, L_check_fill_32_bytes); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8754 |
align(16); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8755 |
|
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8756 |
BIND(L_fill_64_bytes_loop); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8757 |
vmovdqu(Address(to, 0), xtmp); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8758 |
vmovdqu(Address(to, 32), xtmp); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8759 |
addptr(to, 64); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8760 |
subl(count, 16 << shift); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8761 |
jcc(Assembler::greaterEqual, L_fill_64_bytes_loop); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8762 |
|
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8763 |
BIND(L_check_fill_32_bytes); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8764 |
addl(count, 8 << shift); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8765 |
jccb(Assembler::less, L_check_fill_8_bytes); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8766 |
vmovdqu(Address(to, 0), xtmp); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8767 |
addptr(to, 32); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8768 |
subl(count, 8 << shift); |
16624
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
8769 |
|
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
8770 |
BIND(L_check_fill_8_bytes); |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
8771 |
// clean upper bits of YMM registers |
30299 | 8772 |
movdl(xtmp, value); |
8773 |
pshufd(xtmp, xtmp, 0); |
|
14626 | 8774 |
} else { |
15115
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8775 |
// Fill 32-byte chunks |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8776 |
pshufd(xtmp, xtmp, 0); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8777 |
|
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8778 |
subl(count, 8 << shift); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8779 |
jcc(Assembler::less, L_check_fill_8_bytes); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8780 |
align(16); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8781 |
|
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8782 |
BIND(L_fill_32_bytes_loop); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8783 |
|
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8784 |
if (UseUnalignedLoadStores) { |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8785 |
movdqu(Address(to, 0), xtmp); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8786 |
movdqu(Address(to, 16), xtmp); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8787 |
} else { |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8788 |
movq(Address(to, 0), xtmp); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8789 |
movq(Address(to, 8), xtmp); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8790 |
movq(Address(to, 16), xtmp); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8791 |
movq(Address(to, 24), xtmp); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8792 |
} |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8793 |
|
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8794 |
addptr(to, 32); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8795 |
subl(count, 8 << shift); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
8796 |
jcc(Assembler::greaterEqual, L_fill_32_bytes_loop); |
16624
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
8797 |
|
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
8798 |
BIND(L_check_fill_8_bytes); |
14626 | 8799 |
} |
8800 |
addl(count, 8 << shift); |
|
8801 |
jccb(Assembler::zero, L_exit); |
|
8802 |
jmpb(L_fill_8_bytes); |
|
8803 |
||
8804 |
// |
|
8805 |
// length is too short, just fill qwords |
|
8806 |
// |
|
8807 |
BIND(L_fill_8_bytes_loop); |
|
8808 |
movq(Address(to, 0), xtmp); |
|
8809 |
addptr(to, 8); |
|
8810 |
BIND(L_fill_8_bytes); |
|
8811 |
subl(count, 1 << (shift + 1)); |
|
8812 |
jcc(Assembler::greaterEqual, L_fill_8_bytes_loop); |
|
8813 |
} |
|
8814 |
} |
|
8815 |
// fill trailing 4 bytes |
|
8816 |
BIND(L_fill_4_bytes); |
|
8817 |
testl(count, 1<<shift); |
|
8818 |
jccb(Assembler::zero, L_fill_2_bytes); |
|
8819 |
movl(Address(to, 0), value); |
|
8820 |
if (t == T_BYTE || t == T_SHORT) { |
|
8821 |
addptr(to, 4); |
|
8822 |
BIND(L_fill_2_bytes); |
|
8823 |
// fill trailing 2 bytes |
|
8824 |
testl(count, 1<<(shift-1)); |
|
8825 |
jccb(Assembler::zero, L_fill_byte); |
|
8826 |
movw(Address(to, 0), value); |
|
8827 |
if (t == T_BYTE) { |
|
8828 |
addptr(to, 2); |
|
8829 |
BIND(L_fill_byte); |
|
8830 |
// fill trailing byte |
|
8831 |
testl(count, 1); |
|
8832 |
jccb(Assembler::zero, L_exit); |
|
8833 |
movb(Address(to, 0), value); |
|
8834 |
} else { |
|
8835 |
BIND(L_fill_byte); |
|
8836 |
} |
|
8837 |
} else { |
|
8838 |
BIND(L_fill_2_bytes); |
|
8839 |
} |
|
8840 |
BIND(L_exit); |
|
8841 |
} |
|
15242
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8842 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8843 |
// encode char[] to byte[] in ISO_8859_1 |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8844 |
void MacroAssembler::encode_iso_array(Register src, Register dst, Register len, |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8845 |
XMMRegister tmp1Reg, XMMRegister tmp2Reg, |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8846 |
XMMRegister tmp3Reg, XMMRegister tmp4Reg, |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8847 |
Register tmp5, Register result) { |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8848 |
// rsi: src |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8849 |
// rdi: dst |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8850 |
// rdx: len |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8851 |
// rcx: tmp5 |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8852 |
// rax: result |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8853 |
ShortBranchVerifier sbv(this); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8854 |
assert_different_registers(src, dst, len, tmp5, result); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8855 |
Label L_done, L_copy_1_char, L_copy_1_char_exit; |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8856 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8857 |
// set result |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8858 |
xorl(result, result); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8859 |
// check for zero length |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8860 |
testl(len, len); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8861 |
jcc(Assembler::zero, L_done); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8862 |
movl(result, len); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8863 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8864 |
// Setup pointers |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8865 |
lea(src, Address(src, len, Address::times_2)); // char[] |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8866 |
lea(dst, Address(dst, len, Address::times_1)); // byte[] |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8867 |
negptr(len); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8868 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8869 |
if (UseSSE42Intrinsics || UseAVX >= 2) { |
34207
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
8870 |
assert(UseSSE42Intrinsics ? UseSSE >= 4 : true, "SSE4 must be enabled for SSE4.2 intrinsics to be available"); |
15242
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8871 |
Label L_chars_8_check, L_copy_8_chars, L_copy_8_chars_exit; |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8872 |
Label L_chars_16_check, L_copy_16_chars, L_copy_16_chars_exit; |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8873 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8874 |
if (UseAVX >= 2) { |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8875 |
Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit; |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8876 |
movl(tmp5, 0xff00ff00); // create mask to test for Unicode chars in vector |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8877 |
movdl(tmp1Reg, tmp5); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8878 |
vpbroadcastd(tmp1Reg, tmp1Reg); |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
8879 |
jmp(L_chars_32_check); |
15242
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8880 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8881 |
bind(L_copy_32_chars); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8882 |
vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64)); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8883 |
vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32)); |
30624 | 8884 |
vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1); |
15242
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8885 |
vptest(tmp2Reg, tmp1Reg); // check for Unicode chars in vector |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8886 |
jccb(Assembler::notZero, L_copy_32_chars_exit); |
30624 | 8887 |
vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1); |
8888 |
vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector_len */ 1); |
|
15242
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8889 |
vmovdqu(Address(dst, len, Address::times_1, -32), tmp4Reg); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8890 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8891 |
bind(L_chars_32_check); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8892 |
addptr(len, 32); |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
8893 |
jcc(Assembler::lessEqual, L_copy_32_chars); |
15242
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8894 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8895 |
bind(L_copy_32_chars_exit); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8896 |
subptr(len, 16); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8897 |
jccb(Assembler::greater, L_copy_16_chars_exit); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8898 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8899 |
} else if (UseSSE42Intrinsics) { |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8900 |
movl(tmp5, 0xff00ff00); // create mask to test for Unicode chars in vector |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8901 |
movdl(tmp1Reg, tmp5); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8902 |
pshufd(tmp1Reg, tmp1Reg, 0); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8903 |
jmpb(L_chars_16_check); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8904 |
} |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8905 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8906 |
bind(L_copy_16_chars); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8907 |
if (UseAVX >= 2) { |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8908 |
vmovdqu(tmp2Reg, Address(src, len, Address::times_2, -32)); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8909 |
vptest(tmp2Reg, tmp1Reg); |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
8910 |
jcc(Assembler::notZero, L_copy_16_chars_exit); |
30624 | 8911 |
vpackuswb(tmp2Reg, tmp2Reg, tmp1Reg, /* vector_len */ 1); |
8912 |
vpermq(tmp3Reg, tmp2Reg, 0xD8, /* vector_len */ 1); |
|
15242
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8913 |
} else { |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8914 |
if (UseAVX > 0) { |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8915 |
movdqu(tmp3Reg, Address(src, len, Address::times_2, -32)); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8916 |
movdqu(tmp4Reg, Address(src, len, Address::times_2, -16)); |
30624 | 8917 |
vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 0); |
15242
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8918 |
} else { |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8919 |
movdqu(tmp3Reg, Address(src, len, Address::times_2, -32)); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8920 |
por(tmp2Reg, tmp3Reg); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8921 |
movdqu(tmp4Reg, Address(src, len, Address::times_2, -16)); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8922 |
por(tmp2Reg, tmp4Reg); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8923 |
} |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8924 |
ptest(tmp2Reg, tmp1Reg); // check for Unicode chars in vector |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8925 |
jccb(Assembler::notZero, L_copy_16_chars_exit); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8926 |
packuswb(tmp3Reg, tmp4Reg); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8927 |
} |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8928 |
movdqu(Address(dst, len, Address::times_1, -16), tmp3Reg); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8929 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8930 |
bind(L_chars_16_check); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8931 |
addptr(len, 16); |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
8932 |
jcc(Assembler::lessEqual, L_copy_16_chars); |
15242
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8933 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8934 |
bind(L_copy_16_chars_exit); |
16624
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
8935 |
if (UseAVX >= 2) { |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
8936 |
// clean upper bits of YMM registers |
30299 | 8937 |
vpxor(tmp2Reg, tmp2Reg); |
8938 |
vpxor(tmp3Reg, tmp3Reg); |
|
8939 |
vpxor(tmp4Reg, tmp4Reg); |
|
8940 |
movdl(tmp1Reg, tmp5); |
|
8941 |
pshufd(tmp1Reg, tmp1Reg, 0); |
|
16624
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
8942 |
} |
15242
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8943 |
subptr(len, 8); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8944 |
jccb(Assembler::greater, L_copy_8_chars_exit); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8945 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8946 |
bind(L_copy_8_chars); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8947 |
movdqu(tmp3Reg, Address(src, len, Address::times_2, -16)); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8948 |
ptest(tmp3Reg, tmp1Reg); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8949 |
jccb(Assembler::notZero, L_copy_8_chars_exit); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8950 |
packuswb(tmp3Reg, tmp1Reg); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8951 |
movq(Address(dst, len, Address::times_1, -8), tmp3Reg); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8952 |
addptr(len, 8); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8953 |
jccb(Assembler::lessEqual, L_copy_8_chars); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8954 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8955 |
bind(L_copy_8_chars_exit); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8956 |
subptr(len, 8); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8957 |
jccb(Assembler::zero, L_done); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8958 |
} |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8959 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8960 |
bind(L_copy_1_char); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8961 |
load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0)); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8962 |
testl(tmp5, 0xff00); // check if Unicode char |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8963 |
jccb(Assembler::notZero, L_copy_1_char_exit); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8964 |
movb(Address(dst, len, Address::times_1, 0), tmp5); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8965 |
addptr(len, 1); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8966 |
jccb(Assembler::less, L_copy_1_char); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8967 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8968 |
bind(L_copy_1_char_exit); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8969 |
addptr(result, len); // len is negative count of not processed elements |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8970 |
bind(L_done); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8971 |
} |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
8972 |
|
26434
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8973 |
#ifdef _LP64 |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8974 |
/** |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8975 |
* Helper for multiply_to_len(). |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8976 |
*/ |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8977 |
void MacroAssembler::add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2) { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8978 |
addq(dest_lo, src1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8979 |
adcq(dest_hi, 0); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8980 |
addq(dest_lo, src2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8981 |
adcq(dest_hi, 0); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8982 |
} |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8983 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8984 |
/** |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8985 |
* Multiply 64 bit by 64 bit first loop. |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8986 |
*/ |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8987 |
void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart, |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8988 |
Register y, Register y_idx, Register z, |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8989 |
Register carry, Register product, |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8990 |
Register idx, Register kdx) { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8991 |
// |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8992 |
// jlong carry, x[], y[], z[]; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8993 |
// for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8994 |
// huge_128 product = y[idx] * x[xstart] + carry; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8995 |
// z[kdx] = (jlong)product; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8996 |
// carry = (jlong)(product >>> 64); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8997 |
// } |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8998 |
// z[xstart] = carry; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8999 |
// |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9000 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9001 |
Label L_first_loop, L_first_loop_exit; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9002 |
Label L_one_x, L_one_y, L_multiply; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9003 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9004 |
decrementl(xstart); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9005 |
jcc(Assembler::negative, L_one_x); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9006 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9007 |
movq(x_xstart, Address(x, xstart, Address::times_4, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9008 |
rorq(x_xstart, 32); // convert big-endian to little-endian |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9009 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9010 |
bind(L_first_loop); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9011 |
decrementl(idx); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9012 |
jcc(Assembler::negative, L_first_loop_exit); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9013 |
decrementl(idx); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9014 |
jcc(Assembler::negative, L_one_y); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9015 |
movq(y_idx, Address(y, idx, Address::times_4, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9016 |
rorq(y_idx, 32); // convert big-endian to little-endian |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9017 |
bind(L_multiply); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9018 |
movq(product, x_xstart); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9019 |
mulq(y_idx); // product(rax) * y_idx -> rdx:rax |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9020 |
addq(product, carry); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9021 |
adcq(rdx, 0); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9022 |
subl(kdx, 2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9023 |
movl(Address(z, kdx, Address::times_4, 4), product); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9024 |
shrq(product, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9025 |
movl(Address(z, kdx, Address::times_4, 0), product); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9026 |
movq(carry, rdx); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9027 |
jmp(L_first_loop); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9028 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9029 |
bind(L_one_y); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9030 |
movl(y_idx, Address(y, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9031 |
jmp(L_multiply); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9032 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9033 |
bind(L_one_x); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9034 |
movl(x_xstart, Address(x, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9035 |
jmp(L_first_loop); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9036 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9037 |
bind(L_first_loop_exit); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9038 |
} |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9039 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9040 |
/** |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9041 |
* Multiply 64 bit by 64 bit and add 128 bit. |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9042 |
*/ |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9043 |
void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y, Register z, |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9044 |
Register yz_idx, Register idx, |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9045 |
Register carry, Register product, int offset) { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9046 |
// huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9047 |
// z[kdx] = (jlong)product; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9048 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9049 |
movq(yz_idx, Address(y, idx, Address::times_4, offset)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9050 |
rorq(yz_idx, 32); // convert big-endian to little-endian |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9051 |
movq(product, x_xstart); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9052 |
mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax) |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9053 |
movq(yz_idx, Address(z, idx, Address::times_4, offset)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9054 |
rorq(yz_idx, 32); // convert big-endian to little-endian |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9055 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9056 |
add2_with_carry(rdx, product, carry, yz_idx); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9057 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9058 |
movl(Address(z, idx, Address::times_4, offset+4), product); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9059 |
shrq(product, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9060 |
movl(Address(z, idx, Address::times_4, offset), product); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9061 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9062 |
} |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9063 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9064 |
/** |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9065 |
* Multiply 128 bit by 128 bit. Unrolled inner loop. |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9066 |
*/ |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9067 |
void MacroAssembler::multiply_128_x_128_loop(Register x_xstart, Register y, Register z, |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9068 |
Register yz_idx, Register idx, Register jdx, |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9069 |
Register carry, Register product, |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9070 |
Register carry2) { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9071 |
// jlong carry, x[], y[], z[]; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9072 |
// int kdx = ystart+1; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9073 |
// for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9074 |
// huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9075 |
// z[kdx+idx+1] = (jlong)product; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9076 |
// jlong carry2 = (jlong)(product >>> 64); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9077 |
// product = (y[idx] * x_xstart) + z[kdx+idx] + carry2; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9078 |
// z[kdx+idx] = (jlong)product; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9079 |
// carry = (jlong)(product >>> 64); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9080 |
// } |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9081 |
// idx += 2; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9082 |
// if (idx > 0) { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9083 |
// product = (y[idx] * x_xstart) + z[kdx+idx] + carry; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9084 |
// z[kdx+idx] = (jlong)product; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9085 |
// carry = (jlong)(product >>> 64); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9086 |
// } |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9087 |
// |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9088 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9089 |
Label L_third_loop, L_third_loop_exit, L_post_third_loop_done; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9090 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9091 |
movl(jdx, idx); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9092 |
andl(jdx, 0xFFFFFFFC); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9093 |
shrl(jdx, 2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9094 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9095 |
bind(L_third_loop); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9096 |
subl(jdx, 1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9097 |
jcc(Assembler::negative, L_third_loop_exit); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9098 |
subl(idx, 4); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9099 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9100 |
multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 8); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9101 |
movq(carry2, rdx); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9102 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9103 |
multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product, 0); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9104 |
movq(carry, rdx); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9105 |
jmp(L_third_loop); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9106 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9107 |
bind (L_third_loop_exit); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9108 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9109 |
andl (idx, 0x3); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9110 |
jcc(Assembler::zero, L_post_third_loop_done); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9111 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9112 |
Label L_check_1; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9113 |
subl(idx, 2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9114 |
jcc(Assembler::negative, L_check_1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9115 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9116 |
multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 0); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9117 |
movq(carry, rdx); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9118 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9119 |
bind (L_check_1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9120 |
addl (idx, 0x2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9121 |
andl (idx, 0x1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9122 |
subl(idx, 1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9123 |
jcc(Assembler::negative, L_post_third_loop_done); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9124 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9125 |
movl(yz_idx, Address(y, idx, Address::times_4, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9126 |
movq(product, x_xstart); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9127 |
mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax) |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9128 |
movl(yz_idx, Address(z, idx, Address::times_4, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9129 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9130 |
add2_with_carry(rdx, product, yz_idx, carry); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9131 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9132 |
movl(Address(z, idx, Address::times_4, 0), product); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9133 |
shrq(product, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9134 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9135 |
shlq(rdx, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9136 |
orq(product, rdx); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9137 |
movq(carry, product); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9138 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9139 |
bind(L_post_third_loop_done); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9140 |
} |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9141 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9142 |
/** |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9143 |
* Multiply 128 bit by 128 bit using BMI2. Unrolled inner loop. |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9144 |
* |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9145 |
*/ |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9146 |
void MacroAssembler::multiply_128_x_128_bmi2_loop(Register y, Register z, |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9147 |
Register carry, Register carry2, |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9148 |
Register idx, Register jdx, |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9149 |
Register yz_idx1, Register yz_idx2, |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9150 |
Register tmp, Register tmp3, Register tmp4) { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9151 |
assert(UseBMI2Instructions, "should be used only when BMI2 is available"); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9152 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9153 |
// jlong carry, x[], y[], z[]; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9154 |
// int kdx = ystart+1; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9155 |
// for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9156 |
// huge_128 tmp3 = (y[idx+1] * rdx) + z[kdx+idx+1] + carry; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9157 |
// jlong carry2 = (jlong)(tmp3 >>> 64); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9158 |
// huge_128 tmp4 = (y[idx] * rdx) + z[kdx+idx] + carry2; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9159 |
// carry = (jlong)(tmp4 >>> 64); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9160 |
// z[kdx+idx+1] = (jlong)tmp3; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9161 |
// z[kdx+idx] = (jlong)tmp4; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9162 |
// } |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9163 |
// idx += 2; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9164 |
// if (idx > 0) { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9165 |
// yz_idx1 = (y[idx] * rdx) + z[kdx+idx] + carry; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9166 |
// z[kdx+idx] = (jlong)yz_idx1; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9167 |
// carry = (jlong)(yz_idx1 >>> 64); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9168 |
// } |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9169 |
// |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9170 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9171 |
Label L_third_loop, L_third_loop_exit, L_post_third_loop_done; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9172 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9173 |
movl(jdx, idx); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9174 |
andl(jdx, 0xFFFFFFFC); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9175 |
shrl(jdx, 2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9176 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9177 |
bind(L_third_loop); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9178 |
subl(jdx, 1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9179 |
jcc(Assembler::negative, L_third_loop_exit); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9180 |
subl(idx, 4); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9181 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9182 |
movq(yz_idx1, Address(y, idx, Address::times_4, 8)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9183 |
rorxq(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9184 |
movq(yz_idx2, Address(y, idx, Address::times_4, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9185 |
rorxq(yz_idx2, yz_idx2, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9186 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9187 |
mulxq(tmp4, tmp3, yz_idx1); // yz_idx1 * rdx -> tmp4:tmp3 |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9188 |
mulxq(carry2, tmp, yz_idx2); // yz_idx2 * rdx -> carry2:tmp |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9189 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9190 |
movq(yz_idx1, Address(z, idx, Address::times_4, 8)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9191 |
rorxq(yz_idx1, yz_idx1, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9192 |
movq(yz_idx2, Address(z, idx, Address::times_4, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9193 |
rorxq(yz_idx2, yz_idx2, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9194 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9195 |
if (VM_Version::supports_adx()) { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9196 |
adcxq(tmp3, carry); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9197 |
adoxq(tmp3, yz_idx1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9198 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9199 |
adcxq(tmp4, tmp); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9200 |
adoxq(tmp4, yz_idx2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9201 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9202 |
movl(carry, 0); // does not affect flags |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9203 |
adcxq(carry2, carry); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9204 |
adoxq(carry2, carry); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9205 |
} else { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9206 |
add2_with_carry(tmp4, tmp3, carry, yz_idx1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9207 |
add2_with_carry(carry2, tmp4, tmp, yz_idx2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9208 |
} |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9209 |
movq(carry, carry2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9210 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9211 |
movl(Address(z, idx, Address::times_4, 12), tmp3); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9212 |
shrq(tmp3, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9213 |
movl(Address(z, idx, Address::times_4, 8), tmp3); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9214 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9215 |
movl(Address(z, idx, Address::times_4, 4), tmp4); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9216 |
shrq(tmp4, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9217 |
movl(Address(z, idx, Address::times_4, 0), tmp4); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9218 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9219 |
jmp(L_third_loop); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9220 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9221 |
bind (L_third_loop_exit); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9222 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9223 |
andl (idx, 0x3); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9224 |
jcc(Assembler::zero, L_post_third_loop_done); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9225 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9226 |
Label L_check_1; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9227 |
subl(idx, 2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9228 |
jcc(Assembler::negative, L_check_1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9229 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9230 |
movq(yz_idx1, Address(y, idx, Address::times_4, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9231 |
rorxq(yz_idx1, yz_idx1, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9232 |
mulxq(tmp4, tmp3, yz_idx1); // yz_idx1 * rdx -> tmp4:tmp3 |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9233 |
movq(yz_idx2, Address(z, idx, Address::times_4, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9234 |
rorxq(yz_idx2, yz_idx2, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9235 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9236 |
add2_with_carry(tmp4, tmp3, carry, yz_idx2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9237 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9238 |
movl(Address(z, idx, Address::times_4, 4), tmp3); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9239 |
shrq(tmp3, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9240 |
movl(Address(z, idx, Address::times_4, 0), tmp3); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9241 |
movq(carry, tmp4); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9242 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9243 |
bind (L_check_1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9244 |
addl (idx, 0x2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9245 |
andl (idx, 0x1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9246 |
subl(idx, 1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9247 |
jcc(Assembler::negative, L_post_third_loop_done); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9248 |
movl(tmp4, Address(y, idx, Address::times_4, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9249 |
mulxq(carry2, tmp3, tmp4); // tmp4 * rdx -> carry2:tmp3 |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9250 |
movl(tmp4, Address(z, idx, Address::times_4, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9251 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9252 |
add2_with_carry(carry2, tmp3, tmp4, carry); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9253 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9254 |
movl(Address(z, idx, Address::times_4, 0), tmp3); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9255 |
shrq(tmp3, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9256 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9257 |
shlq(carry2, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9258 |
orq(tmp3, carry2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9259 |
movq(carry, tmp3); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9260 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9261 |
bind(L_post_third_loop_done); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9262 |
} |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9263 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9264 |
/** |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9265 |
* Code for BigInteger::multiplyToLen() instrinsic. |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9266 |
* |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9267 |
* rdi: x |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9268 |
* rax: xlen |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9269 |
* rsi: y |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9270 |
* rcx: ylen |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9271 |
* r8: z |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9272 |
* r11: zlen |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9273 |
* r12: tmp1 |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9274 |
* r13: tmp2 |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9275 |
* r14: tmp3 |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9276 |
* r15: tmp4 |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9277 |
* rbx: tmp5 |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9278 |
* |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9279 |
*/ |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9280 |
void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register zlen, |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9281 |
Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9282 |
ShortBranchVerifier sbv(this); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9283 |
assert_different_registers(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, rdx); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9284 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9285 |
push(tmp1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9286 |
push(tmp2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9287 |
push(tmp3); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9288 |
push(tmp4); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9289 |
push(tmp5); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9290 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9291 |
push(xlen); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9292 |
push(zlen); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9293 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9294 |
const Register idx = tmp1; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9295 |
const Register kdx = tmp2; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9296 |
const Register xstart = tmp3; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9297 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9298 |
const Register y_idx = tmp4; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9299 |
const Register carry = tmp5; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9300 |
const Register product = xlen; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9301 |
const Register x_xstart = zlen; // reuse register |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9302 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9303 |
// First Loop. |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9304 |
// |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9305 |
// final static long LONG_MASK = 0xffffffffL; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9306 |
// int xstart = xlen - 1; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9307 |
// int ystart = ylen - 1; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9308 |
// long carry = 0; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9309 |
// for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9310 |
// long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9311 |
// z[kdx] = (int)product; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9312 |
// carry = product >>> 32; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9313 |
// } |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9314 |
// z[xstart] = (int)carry; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9315 |
// |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9316 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9317 |
movl(idx, ylen); // idx = ylen; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9318 |
movl(kdx, zlen); // kdx = xlen+ylen; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9319 |
xorq(carry, carry); // carry = 0; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9320 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9321 |
Label L_done; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9322 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9323 |
movl(xstart, xlen); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9324 |
decrementl(xstart); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9325 |
jcc(Assembler::negative, L_done); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9326 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9327 |
multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9328 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9329 |
Label L_second_loop; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9330 |
testl(kdx, kdx); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9331 |
jcc(Assembler::zero, L_second_loop); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9332 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9333 |
Label L_carry; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9334 |
subl(kdx, 1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9335 |
jcc(Assembler::zero, L_carry); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9336 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9337 |
movl(Address(z, kdx, Address::times_4, 0), carry); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9338 |
shrq(carry, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9339 |
subl(kdx, 1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9340 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9341 |
bind(L_carry); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9342 |
movl(Address(z, kdx, Address::times_4, 0), carry); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9343 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9344 |
// Second and third (nested) loops. |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9345 |
// |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9346 |
// for (int i = xstart-1; i >= 0; i--) { // Second loop |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9347 |
// carry = 0; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9348 |
// for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9349 |
// long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) + |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9350 |
// (z[k] & LONG_MASK) + carry; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9351 |
// z[k] = (int)product; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9352 |
// carry = product >>> 32; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9353 |
// } |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9354 |
// z[i] = (int)carry; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9355 |
// } |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9356 |
// |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9357 |
// i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9358 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9359 |
const Register jdx = tmp1; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9360 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9361 |
bind(L_second_loop); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9362 |
xorl(carry, carry); // carry = 0; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9363 |
movl(jdx, ylen); // j = ystart+1 |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9364 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9365 |
subl(xstart, 1); // i = xstart-1; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9366 |
jcc(Assembler::negative, L_done); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9367 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9368 |
push (z); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9369 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9370 |
Label L_last_x; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9371 |
lea(z, Address(z, xstart, Address::times_4, 4)); // z = z + k - j |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9372 |
subl(xstart, 1); // i = xstart-1; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9373 |
jcc(Assembler::negative, L_last_x); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9374 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9375 |
if (UseBMI2Instructions) { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9376 |
movq(rdx, Address(x, xstart, Address::times_4, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9377 |
rorxq(rdx, rdx, 32); // convert big-endian to little-endian |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9378 |
} else { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9379 |
movq(x_xstart, Address(x, xstart, Address::times_4, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9380 |
rorq(x_xstart, 32); // convert big-endian to little-endian |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9381 |
} |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9382 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9383 |
Label L_third_loop_prologue; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9384 |
bind(L_third_loop_prologue); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9385 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9386 |
push (x); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9387 |
push (xstart); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9388 |
push (ylen); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9389 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9390 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9391 |
if (UseBMI2Instructions) { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9392 |
multiply_128_x_128_bmi2_loop(y, z, carry, x, jdx, ylen, product, tmp2, x_xstart, tmp3, tmp4); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9393 |
} else { // !UseBMI2Instructions |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9394 |
multiply_128_x_128_loop(x_xstart, y, z, y_idx, jdx, ylen, carry, product, x); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9395 |
} |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9396 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9397 |
pop(ylen); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9398 |
pop(xlen); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9399 |
pop(x); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9400 |
pop(z); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9401 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9402 |
movl(tmp3, xlen); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9403 |
addl(tmp3, 1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9404 |
movl(Address(z, tmp3, Address::times_4, 0), carry); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9405 |
subl(tmp3, 1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9406 |
jccb(Assembler::negative, L_done); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9407 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9408 |
shrq(carry, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9409 |
movl(Address(z, tmp3, Address::times_4, 0), carry); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9410 |
jmp(L_second_loop); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9411 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9412 |
// Next infrequent code is moved outside loops. |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9413 |
bind(L_last_x); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9414 |
if (UseBMI2Instructions) { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9415 |
movl(rdx, Address(x, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9416 |
} else { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9417 |
movl(x_xstart, Address(x, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9418 |
} |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9419 |
jmp(L_third_loop_prologue); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9420 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9421 |
bind(L_done); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9422 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9423 |
pop(zlen); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9424 |
pop(xlen); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9425 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9426 |
pop(tmp5); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9427 |
pop(tmp4); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9428 |
pop(tmp3); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9429 |
pop(tmp2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9430 |
pop(tmp1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
9431 |
} |
31129
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9432 |
|
35110
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9433 |
void MacroAssembler::vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale, |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9434 |
Register result, Register tmp1, Register tmp2, XMMRegister rymm0, XMMRegister rymm1, XMMRegister rymm2){ |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9435 |
assert(UseSSE42Intrinsics, "SSE4.2 must be enabled."); |
38138
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9436 |
Label VECTOR64_LOOP, VECTOR64_TAIL, VECTOR64_NOT_EQUAL, VECTOR32_TAIL; |
35110
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9437 |
Label VECTOR32_LOOP, VECTOR16_LOOP, VECTOR8_LOOP, VECTOR4_LOOP; |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9438 |
Label VECTOR16_TAIL, VECTOR8_TAIL, VECTOR4_TAIL; |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9439 |
Label VECTOR32_NOT_EQUAL, VECTOR16_NOT_EQUAL, VECTOR8_NOT_EQUAL, VECTOR4_NOT_EQUAL; |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9440 |
Label SAME_TILL_END, DONE; |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9441 |
Label BYTES_LOOP, BYTES_TAIL, BYTES_NOT_EQUAL; |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9442 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9443 |
//scale is in rcx in both Win64 and Unix |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9444 |
ShortBranchVerifier sbv(this); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9445 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9446 |
shlq(length); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9447 |
xorq(result, result); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9448 |
|
38138
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9449 |
if ((UseAVX > 2) && |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9450 |
VM_Version::supports_avx512vlbw()) { |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9451 |
set_vector_masking(); // opening of the stub context for programming mask registers |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9452 |
cmpq(length, 64); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9453 |
jcc(Assembler::less, VECTOR32_TAIL); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9454 |
movq(tmp1, length); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9455 |
andq(tmp1, 0x3F); // tail count |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9456 |
andq(length, ~(0x3F)); //vector count |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9457 |
|
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9458 |
bind(VECTOR64_LOOP); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9459 |
// AVX512 code to compare 64 byte vectors. |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9460 |
evmovdqub(rymm0, Address(obja, result), Assembler::AVX_512bit); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9461 |
evpcmpeqb(k7, rymm0, Address(objb, result), Assembler::AVX_512bit); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9462 |
kortestql(k7, k7); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9463 |
jcc(Assembler::aboveEqual, VECTOR64_NOT_EQUAL); // mismatch |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9464 |
addq(result, 64); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9465 |
subq(length, 64); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9466 |
jccb(Assembler::notZero, VECTOR64_LOOP); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9467 |
|
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9468 |
//bind(VECTOR64_TAIL); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9469 |
testq(tmp1, tmp1); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9470 |
jcc(Assembler::zero, SAME_TILL_END); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9471 |
|
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9472 |
bind(VECTOR64_TAIL); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9473 |
// AVX512 code to compare upto 63 byte vectors. |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9474 |
// Save k1 |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9475 |
kmovql(k3, k1); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9476 |
mov64(tmp2, 0xFFFFFFFFFFFFFFFF); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9477 |
shlxq(tmp2, tmp2, tmp1); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9478 |
notq(tmp2); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9479 |
kmovql(k1, tmp2); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9480 |
|
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9481 |
evmovdqub(k1, rymm0, Address(obja, result), Assembler::AVX_512bit); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9482 |
evpcmpeqb(k1, k7, rymm0, Address(objb, result), Assembler::AVX_512bit); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9483 |
|
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9484 |
ktestql(k7, k1); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9485 |
// Restore k1 |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9486 |
kmovql(k1, k3); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9487 |
jcc(Assembler::below, SAME_TILL_END); // not mismatch |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9488 |
|
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9489 |
bind(VECTOR64_NOT_EQUAL); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9490 |
kmovql(tmp1, k7); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9491 |
notq(tmp1); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9492 |
tzcntq(tmp1, tmp1); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9493 |
addq(result, tmp1); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9494 |
shrq(result); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9495 |
jmp(DONE); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9496 |
bind(VECTOR32_TAIL); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9497 |
clear_vector_masking(); // closing of the stub context for programming mask registers |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9498 |
} |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9499 |
|
35110
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9500 |
cmpq(length, 8); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9501 |
jcc(Assembler::equal, VECTOR8_LOOP); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9502 |
jcc(Assembler::less, VECTOR4_TAIL); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9503 |
|
38138
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9504 |
if (UseAVX >= 2) { |
35110
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9505 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9506 |
cmpq(length, 16); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9507 |
jcc(Assembler::equal, VECTOR16_LOOP); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9508 |
jcc(Assembler::less, VECTOR8_LOOP); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9509 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9510 |
cmpq(length, 32); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9511 |
jccb(Assembler::less, VECTOR16_TAIL); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9512 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9513 |
subq(length, 32); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9514 |
bind(VECTOR32_LOOP); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9515 |
vmovdqu(rymm0, Address(obja, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9516 |
vmovdqu(rymm1, Address(objb, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9517 |
vpxor(rymm2, rymm0, rymm1, Assembler::AVX_256bit); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9518 |
vptest(rymm2, rymm2); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9519 |
jcc(Assembler::notZero, VECTOR32_NOT_EQUAL);//mismatch found |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9520 |
addq(result, 32); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9521 |
subq(length, 32); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9522 |
jccb(Assembler::greaterEqual, VECTOR32_LOOP); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9523 |
addq(length, 32); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9524 |
jcc(Assembler::equal, SAME_TILL_END); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9525 |
//falling through if less than 32 bytes left //close the branch here. |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9526 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9527 |
bind(VECTOR16_TAIL); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9528 |
cmpq(length, 16); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9529 |
jccb(Assembler::less, VECTOR8_TAIL); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9530 |
bind(VECTOR16_LOOP); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9531 |
movdqu(rymm0, Address(obja, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9532 |
movdqu(rymm1, Address(objb, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9533 |
vpxor(rymm2, rymm0, rymm1, Assembler::AVX_128bit); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9534 |
ptest(rymm2, rymm2); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9535 |
jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9536 |
addq(result, 16); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9537 |
subq(length, 16); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9538 |
jcc(Assembler::equal, SAME_TILL_END); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9539 |
//falling through if less than 16 bytes left |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9540 |
} else {//regular intrinsics |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9541 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9542 |
cmpq(length, 16); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9543 |
jccb(Assembler::less, VECTOR8_TAIL); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9544 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9545 |
subq(length, 16); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9546 |
bind(VECTOR16_LOOP); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9547 |
movdqu(rymm0, Address(obja, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9548 |
movdqu(rymm1, Address(objb, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9549 |
pxor(rymm0, rymm1); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9550 |
ptest(rymm0, rymm0); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9551 |
jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9552 |
addq(result, 16); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9553 |
subq(length, 16); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9554 |
jccb(Assembler::greaterEqual, VECTOR16_LOOP); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9555 |
addq(length, 16); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9556 |
jcc(Assembler::equal, SAME_TILL_END); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9557 |
//falling through if less than 16 bytes left |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9558 |
} |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9559 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9560 |
bind(VECTOR8_TAIL); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9561 |
cmpq(length, 8); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9562 |
jccb(Assembler::less, VECTOR4_TAIL); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9563 |
bind(VECTOR8_LOOP); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9564 |
movq(tmp1, Address(obja, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9565 |
movq(tmp2, Address(objb, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9566 |
xorq(tmp1, tmp2); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9567 |
testq(tmp1, tmp1); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9568 |
jcc(Assembler::notZero, VECTOR8_NOT_EQUAL);//mismatch found |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9569 |
addq(result, 8); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9570 |
subq(length, 8); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9571 |
jcc(Assembler::equal, SAME_TILL_END); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9572 |
//falling through if less than 8 bytes left |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9573 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9574 |
bind(VECTOR4_TAIL); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9575 |
cmpq(length, 4); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9576 |
jccb(Assembler::less, BYTES_TAIL); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9577 |
bind(VECTOR4_LOOP); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9578 |
movl(tmp1, Address(obja, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9579 |
xorl(tmp1, Address(objb, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9580 |
testl(tmp1, tmp1); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9581 |
jcc(Assembler::notZero, VECTOR4_NOT_EQUAL);//mismatch found |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9582 |
addq(result, 4); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9583 |
subq(length, 4); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9584 |
jcc(Assembler::equal, SAME_TILL_END); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9585 |
//falling through if less than 4 bytes left |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9586 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9587 |
bind(BYTES_TAIL); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9588 |
bind(BYTES_LOOP); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9589 |
load_unsigned_byte(tmp1, Address(obja, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9590 |
load_unsigned_byte(tmp2, Address(objb, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9591 |
xorl(tmp1, tmp2); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9592 |
testl(tmp1, tmp1); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9593 |
jccb(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9594 |
decq(length); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9595 |
jccb(Assembler::zero, SAME_TILL_END); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9596 |
incq(result); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9597 |
load_unsigned_byte(tmp1, Address(obja, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9598 |
load_unsigned_byte(tmp2, Address(objb, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9599 |
xorl(tmp1, tmp2); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9600 |
testl(tmp1, tmp1); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9601 |
jccb(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9602 |
decq(length); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9603 |
jccb(Assembler::zero, SAME_TILL_END); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9604 |
incq(result); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9605 |
load_unsigned_byte(tmp1, Address(obja, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9606 |
load_unsigned_byte(tmp2, Address(objb, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9607 |
xorl(tmp1, tmp2); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9608 |
testl(tmp1, tmp1); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9609 |
jccb(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9610 |
jmpb(SAME_TILL_END); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9611 |
|
38138
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9612 |
if (UseAVX >= 2) { |
35110
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9613 |
bind(VECTOR32_NOT_EQUAL); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9614 |
vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_256bit); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9615 |
vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_256bit); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9616 |
vpxor(rymm0, rymm0, rymm2, Assembler::AVX_256bit); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9617 |
vpmovmskb(tmp1, rymm0); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9618 |
bsfq(tmp1, tmp1); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9619 |
addq(result, tmp1); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9620 |
shrq(result); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9621 |
jmpb(DONE); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9622 |
} |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9623 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9624 |
bind(VECTOR16_NOT_EQUAL); |
38138
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
9625 |
if (UseAVX >= 2) { |
35110
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9626 |
vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_128bit); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9627 |
vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_128bit); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9628 |
pxor(rymm0, rymm2); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9629 |
} else { |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9630 |
pcmpeqb(rymm2, rymm2); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9631 |
pxor(rymm0, rymm1); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9632 |
pcmpeqb(rymm0, rymm1); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9633 |
pxor(rymm0, rymm2); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9634 |
} |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9635 |
pmovmskb(tmp1, rymm0); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9636 |
bsfq(tmp1, tmp1); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9637 |
addq(result, tmp1); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9638 |
shrq(result); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9639 |
jmpb(DONE); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9640 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9641 |
bind(VECTOR8_NOT_EQUAL); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9642 |
bind(VECTOR4_NOT_EQUAL); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9643 |
bsfq(tmp1, tmp1); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9644 |
shrq(tmp1, 3); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9645 |
addq(result, tmp1); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9646 |
bind(BYTES_NOT_EQUAL); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9647 |
shrq(result); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9648 |
jmpb(DONE); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9649 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9650 |
bind(SAME_TILL_END); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9651 |
mov64(result, -1); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9652 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9653 |
bind(DONE); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9654 |
} |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
9655 |
|
31129
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9656 |
//Helper functions for square_to_len() |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9657 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9658 |
/** |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9659 |
* Store the squares of x[], right shifted one bit (divided by 2) into z[] |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9660 |
* Preserves x and z and modifies rest of the registers. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9661 |
*/ |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9662 |
void MacroAssembler::square_rshift(Register x, Register xlen, Register z, Register tmp1, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9663 |
// Perform square and right shift by 1 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9664 |
// Handle odd xlen case first, then for even xlen do the following |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9665 |
// jlong carry = 0; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9666 |
// for (int j=0, i=0; j < xlen; j+=2, i+=4) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9667 |
// huge_128 product = x[j:j+1] * x[j:j+1]; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9668 |
// z[i:i+1] = (carry << 63) | (jlong)(product >>> 65); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9669 |
// z[i+2:i+3] = (jlong)(product >>> 1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9670 |
// carry = (jlong)product; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9671 |
// } |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9672 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9673 |
xorq(tmp5, tmp5); // carry |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9674 |
xorq(rdxReg, rdxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9675 |
xorl(tmp1, tmp1); // index for x |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9676 |
xorl(tmp4, tmp4); // index for z |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9677 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9678 |
Label L_first_loop, L_first_loop_exit; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9679 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9680 |
testl(xlen, 1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9681 |
jccb(Assembler::zero, L_first_loop); //jump if xlen is even |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9682 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9683 |
// Square and right shift by 1 the odd element using 32 bit multiply |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9684 |
movl(raxReg, Address(x, tmp1, Address::times_4, 0)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9685 |
imulq(raxReg, raxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9686 |
shrq(raxReg, 1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9687 |
adcq(tmp5, 0); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9688 |
movq(Address(z, tmp4, Address::times_4, 0), raxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9689 |
incrementl(tmp1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9690 |
addl(tmp4, 2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9691 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9692 |
// Square and right shift by 1 the rest using 64 bit multiply |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9693 |
bind(L_first_loop); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9694 |
cmpptr(tmp1, xlen); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9695 |
jccb(Assembler::equal, L_first_loop_exit); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9696 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9697 |
// Square |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9698 |
movq(raxReg, Address(x, tmp1, Address::times_4, 0)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9699 |
rorq(raxReg, 32); // convert big-endian to little-endian |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9700 |
mulq(raxReg); // 64-bit multiply rax * rax -> rdx:rax |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9701 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9702 |
// Right shift by 1 and save carry |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9703 |
shrq(tmp5, 1); // rdx:rax:tmp5 = (tmp5:rdx:rax) >>> 1 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9704 |
rcrq(rdxReg, 1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9705 |
rcrq(raxReg, 1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9706 |
adcq(tmp5, 0); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9707 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9708 |
// Store result in z |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9709 |
movq(Address(z, tmp4, Address::times_4, 0), rdxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9710 |
movq(Address(z, tmp4, Address::times_4, 8), raxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9711 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9712 |
// Update indices for x and z |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9713 |
addl(tmp1, 2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9714 |
addl(tmp4, 4); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9715 |
jmp(L_first_loop); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9716 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9717 |
bind(L_first_loop_exit); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9718 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9719 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9720 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9721 |
/** |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9722 |
* Perform the following multiply add operation using BMI2 instructions |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9723 |
* carry:sum = sum + op1*op2 + carry |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9724 |
* op2 should be in rdx |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9725 |
* op2 is preserved, all other registers are modified |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9726 |
*/ |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9727 |
void MacroAssembler::multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry, Register tmp2) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9728 |
// assert op2 is rdx |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9729 |
mulxq(tmp2, op1, op1); // op1 * op2 -> tmp2:op1 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9730 |
addq(sum, carry); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9731 |
adcq(tmp2, 0); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9732 |
addq(sum, op1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9733 |
adcq(tmp2, 0); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9734 |
movq(carry, tmp2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9735 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9736 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9737 |
/** |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9738 |
* Perform the following multiply add operation: |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9739 |
* carry:sum = sum + op1*op2 + carry |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9740 |
* Preserves op1, op2 and modifies rest of registers |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9741 |
*/ |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9742 |
void MacroAssembler::multiply_add_64(Register sum, Register op1, Register op2, Register carry, Register rdxReg, Register raxReg) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9743 |
// rdx:rax = op1 * op2 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9744 |
movq(raxReg, op2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9745 |
mulq(op1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9746 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9747 |
// rdx:rax = sum + carry + rdx:rax |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9748 |
addq(sum, carry); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9749 |
adcq(rdxReg, 0); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9750 |
addq(sum, raxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9751 |
adcq(rdxReg, 0); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9752 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9753 |
// carry:sum = rdx:sum |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9754 |
movq(carry, rdxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9755 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9756 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9757 |
/** |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9758 |
* Add 64 bit long carry into z[] with carry propogation. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9759 |
* Preserves z and carry register values and modifies rest of registers. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9760 |
* |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9761 |
*/ |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9762 |
void MacroAssembler::add_one_64(Register z, Register zlen, Register carry, Register tmp1) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9763 |
Label L_fourth_loop, L_fourth_loop_exit; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9764 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9765 |
movl(tmp1, 1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9766 |
subl(zlen, 2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9767 |
addq(Address(z, zlen, Address::times_4, 0), carry); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9768 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9769 |
bind(L_fourth_loop); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9770 |
jccb(Assembler::carryClear, L_fourth_loop_exit); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9771 |
subl(zlen, 2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9772 |
jccb(Assembler::negative, L_fourth_loop_exit); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9773 |
addq(Address(z, zlen, Address::times_4, 0), tmp1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9774 |
jmp(L_fourth_loop); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9775 |
bind(L_fourth_loop_exit); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9776 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9777 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9778 |
/** |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9779 |
* Shift z[] left by 1 bit. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9780 |
* Preserves x, len, z and zlen registers and modifies rest of the registers. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9781 |
* |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9782 |
*/ |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9783 |
void MacroAssembler::lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9784 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9785 |
Label L_fifth_loop, L_fifth_loop_exit; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9786 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9787 |
// Fifth loop |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9788 |
// Perform primitiveLeftShift(z, zlen, 1) |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9789 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9790 |
const Register prev_carry = tmp1; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9791 |
const Register new_carry = tmp4; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9792 |
const Register value = tmp2; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9793 |
const Register zidx = tmp3; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9794 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9795 |
// int zidx, carry; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9796 |
// long value; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9797 |
// carry = 0; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9798 |
// for (zidx = zlen-2; zidx >=0; zidx -= 2) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9799 |
// (carry:value) = (z[i] << 1) | carry ; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9800 |
// z[i] = value; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9801 |
// } |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9802 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9803 |
movl(zidx, zlen); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9804 |
xorl(prev_carry, prev_carry); // clear carry flag and prev_carry register |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9805 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9806 |
bind(L_fifth_loop); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9807 |
decl(zidx); // Use decl to preserve carry flag |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9808 |
decl(zidx); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9809 |
jccb(Assembler::negative, L_fifth_loop_exit); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9810 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9811 |
if (UseBMI2Instructions) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9812 |
movq(value, Address(z, zidx, Address::times_4, 0)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9813 |
rclq(value, 1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9814 |
rorxq(value, value, 32); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9815 |
movq(Address(z, zidx, Address::times_4, 0), value); // Store back in big endian form |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9816 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9817 |
else { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9818 |
// clear new_carry |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9819 |
xorl(new_carry, new_carry); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9820 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9821 |
// Shift z[i] by 1, or in previous carry and save new carry |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9822 |
movq(value, Address(z, zidx, Address::times_4, 0)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9823 |
shlq(value, 1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9824 |
adcl(new_carry, 0); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9825 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9826 |
orq(value, prev_carry); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9827 |
rorq(value, 0x20); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9828 |
movq(Address(z, zidx, Address::times_4, 0), value); // Store back in big endian form |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9829 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9830 |
// Set previous carry = new carry |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9831 |
movl(prev_carry, new_carry); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9832 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9833 |
jmp(L_fifth_loop); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9834 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9835 |
bind(L_fifth_loop_exit); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9836 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9837 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9838 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9839 |
/** |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9840 |
* Code for BigInteger::squareToLen() intrinsic |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9841 |
* |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9842 |
* rdi: x |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9843 |
* rsi: len |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9844 |
* r8: z |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9845 |
* rcx: zlen |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9846 |
* r12: tmp1 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9847 |
* r13: tmp2 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9848 |
* r14: tmp3 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9849 |
* r15: tmp4 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9850 |
* rbx: tmp5 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9851 |
* |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9852 |
*/ |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9853 |
void MacroAssembler::square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9854 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9855 |
Label L_second_loop, L_second_loop_exit, L_third_loop, L_third_loop_exit, fifth_loop, fifth_loop_exit, L_last_x, L_multiply; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9856 |
push(tmp1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9857 |
push(tmp2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9858 |
push(tmp3); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9859 |
push(tmp4); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9860 |
push(tmp5); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9861 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9862 |
// First loop |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9863 |
// Store the squares, right shifted one bit (i.e., divided by 2). |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9864 |
square_rshift(x, len, z, tmp1, tmp3, tmp4, tmp5, rdxReg, raxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9865 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9866 |
// Add in off-diagonal sums. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9867 |
// |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9868 |
// Second, third (nested) and fourth loops. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9869 |
// zlen +=2; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9870 |
// for (int xidx=len-2,zidx=zlen-4; xidx > 0; xidx-=2,zidx-=4) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9871 |
// carry = 0; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9872 |
// long op2 = x[xidx:xidx+1]; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9873 |
// for (int j=xidx-2,k=zidx; j >= 0; j-=2) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9874 |
// k -= 2; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9875 |
// long op1 = x[j:j+1]; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9876 |
// long sum = z[k:k+1]; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9877 |
// carry:sum = multiply_add_64(sum, op1, op2, carry, tmp_regs); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9878 |
// z[k:k+1] = sum; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9879 |
// } |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9880 |
// add_one_64(z, k, carry, tmp_regs); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9881 |
// } |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9882 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9883 |
const Register carry = tmp5; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9884 |
const Register sum = tmp3; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9885 |
const Register op1 = tmp4; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9886 |
Register op2 = tmp2; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9887 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9888 |
push(zlen); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9889 |
push(len); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9890 |
addl(zlen,2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9891 |
bind(L_second_loop); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9892 |
xorq(carry, carry); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9893 |
subl(zlen, 4); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9894 |
subl(len, 2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9895 |
push(zlen); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9896 |
push(len); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9897 |
cmpl(len, 0); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9898 |
jccb(Assembler::lessEqual, L_second_loop_exit); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9899 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9900 |
// Multiply an array by one 64 bit long. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9901 |
if (UseBMI2Instructions) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9902 |
op2 = rdxReg; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9903 |
movq(op2, Address(x, len, Address::times_4, 0)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9904 |
rorxq(op2, op2, 32); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9905 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9906 |
else { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9907 |
movq(op2, Address(x, len, Address::times_4, 0)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9908 |
rorq(op2, 32); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9909 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9910 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9911 |
bind(L_third_loop); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9912 |
decrementl(len); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9913 |
jccb(Assembler::negative, L_third_loop_exit); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9914 |
decrementl(len); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9915 |
jccb(Assembler::negative, L_last_x); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9916 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9917 |
movq(op1, Address(x, len, Address::times_4, 0)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9918 |
rorq(op1, 32); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9919 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9920 |
bind(L_multiply); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9921 |
subl(zlen, 2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9922 |
movq(sum, Address(z, zlen, Address::times_4, 0)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9923 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9924 |
// Multiply 64 bit by 64 bit and add 64 bits lower half and upper 64 bits as carry. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9925 |
if (UseBMI2Instructions) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9926 |
multiply_add_64_bmi2(sum, op1, op2, carry, tmp2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9927 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9928 |
else { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9929 |
multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9930 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9931 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9932 |
movq(Address(z, zlen, Address::times_4, 0), sum); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9933 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9934 |
jmp(L_third_loop); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9935 |
bind(L_third_loop_exit); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9936 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9937 |
// Fourth loop |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9938 |
// Add 64 bit long carry into z with carry propogation. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9939 |
// Uses offsetted zlen. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9940 |
add_one_64(z, zlen, carry, tmp1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9941 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9942 |
pop(len); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9943 |
pop(zlen); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9944 |
jmp(L_second_loop); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9945 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9946 |
// Next infrequent code is moved outside loops. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9947 |
bind(L_last_x); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9948 |
movl(op1, Address(x, 0)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9949 |
jmp(L_multiply); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9950 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9951 |
bind(L_second_loop_exit); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9952 |
pop(len); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9953 |
pop(zlen); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9954 |
pop(len); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9955 |
pop(zlen); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9956 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9957 |
// Fifth loop |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9958 |
// Shift z left 1 bit. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9959 |
lshift_by_1(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9960 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9961 |
// z[zlen-1] |= x[len-1] & 1; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9962 |
movl(tmp3, Address(x, len, Address::times_4, -4)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9963 |
andl(tmp3, 1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9964 |
orl(Address(z, zlen, Address::times_4, -4), tmp3); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9965 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9966 |
pop(tmp5); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9967 |
pop(tmp4); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9968 |
pop(tmp3); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9969 |
pop(tmp2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9970 |
pop(tmp1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9971 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9972 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9973 |
/** |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9974 |
* Helper function for mul_add() |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9975 |
* Multiply the in[] by int k and add to out[] starting at offset offs using |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9976 |
* 128 bit by 32 bit multiply and return the carry in tmp5. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9977 |
* Only quad int aligned length of in[] is operated on in this function. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9978 |
* k is in rdxReg for BMI2Instructions, for others it is in tmp2. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9979 |
* This function preserves out, in and k registers. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9980 |
* len and offset point to the appropriate index in "in" & "out" correspondingly |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9981 |
* tmp5 has the carry. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9982 |
* other registers are temporary and are modified. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9983 |
* |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9984 |
*/ |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9985 |
void MacroAssembler::mul_add_128_x_32_loop(Register out, Register in, |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9986 |
Register offset, Register len, Register tmp1, Register tmp2, Register tmp3, |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9987 |
Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9988 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9989 |
Label L_first_loop, L_first_loop_exit; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9990 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9991 |
movl(tmp1, len); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9992 |
shrl(tmp1, 2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9993 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9994 |
bind(L_first_loop); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9995 |
subl(tmp1, 1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9996 |
jccb(Assembler::negative, L_first_loop_exit); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9997 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9998 |
subl(len, 4); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
9999 |
subl(offset, 4); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10000 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10001 |
Register op2 = tmp2; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10002 |
const Register sum = tmp3; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10003 |
const Register op1 = tmp4; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10004 |
const Register carry = tmp5; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10005 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10006 |
if (UseBMI2Instructions) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10007 |
op2 = rdxReg; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10008 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10009 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10010 |
movq(op1, Address(in, len, Address::times_4, 8)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10011 |
rorq(op1, 32); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10012 |
movq(sum, Address(out, offset, Address::times_4, 8)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10013 |
rorq(sum, 32); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10014 |
if (UseBMI2Instructions) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10015 |
multiply_add_64_bmi2(sum, op1, op2, carry, raxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10016 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10017 |
else { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10018 |
multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10019 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10020 |
// Store back in big endian from little endian |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10021 |
rorq(sum, 0x20); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10022 |
movq(Address(out, offset, Address::times_4, 8), sum); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10023 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10024 |
movq(op1, Address(in, len, Address::times_4, 0)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10025 |
rorq(op1, 32); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10026 |
movq(sum, Address(out, offset, Address::times_4, 0)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10027 |
rorq(sum, 32); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10028 |
if (UseBMI2Instructions) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10029 |
multiply_add_64_bmi2(sum, op1, op2, carry, raxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10030 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10031 |
else { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10032 |
multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10033 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10034 |
// Store back in big endian from little endian |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10035 |
rorq(sum, 0x20); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10036 |
movq(Address(out, offset, Address::times_4, 0), sum); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10037 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10038 |
jmp(L_first_loop); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10039 |
bind(L_first_loop_exit); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10040 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10041 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10042 |
/** |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10043 |
* Code for BigInteger::mulAdd() intrinsic |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10044 |
* |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10045 |
* rdi: out |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10046 |
* rsi: in |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10047 |
* r11: offs (out.length - offset) |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10048 |
* rcx: len |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10049 |
* r8: k |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10050 |
* r12: tmp1 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10051 |
* r13: tmp2 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10052 |
* r14: tmp3 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10053 |
* r15: tmp4 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10054 |
* rbx: tmp5 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10055 |
* Multiply the in[] by word k and add to out[], return the carry in rax |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10056 |
*/ |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10057 |
void MacroAssembler::mul_add(Register out, Register in, Register offs, |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10058 |
Register len, Register k, Register tmp1, Register tmp2, Register tmp3, |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10059 |
Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10060 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10061 |
Label L_carry, L_last_in, L_done; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10062 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10063 |
// carry = 0; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10064 |
// for (int j=len-1; j >= 0; j--) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10065 |
// long product = (in[j] & LONG_MASK) * kLong + |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10066 |
// (out[offs] & LONG_MASK) + carry; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10067 |
// out[offs--] = (int)product; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10068 |
// carry = product >>> 32; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10069 |
// } |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10070 |
// |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10071 |
push(tmp1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10072 |
push(tmp2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10073 |
push(tmp3); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10074 |
push(tmp4); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10075 |
push(tmp5); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10076 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10077 |
Register op2 = tmp2; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10078 |
const Register sum = tmp3; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10079 |
const Register op1 = tmp4; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10080 |
const Register carry = tmp5; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10081 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10082 |
if (UseBMI2Instructions) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10083 |
op2 = rdxReg; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10084 |
movl(op2, k); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10085 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10086 |
else { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10087 |
movl(op2, k); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10088 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10089 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10090 |
xorq(carry, carry); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10091 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10092 |
//First loop |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10093 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10094 |
//Multiply in[] by k in a 4 way unrolled loop using 128 bit by 32 bit multiply |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10095 |
//The carry is in tmp5 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10096 |
mul_add_128_x_32_loop(out, in, offs, len, tmp1, tmp2, tmp3, tmp4, tmp5, rdxReg, raxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10097 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10098 |
//Multiply the trailing in[] entry using 64 bit by 32 bit, if any |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10099 |
decrementl(len); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10100 |
jccb(Assembler::negative, L_carry); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10101 |
decrementl(len); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10102 |
jccb(Assembler::negative, L_last_in); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10103 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10104 |
movq(op1, Address(in, len, Address::times_4, 0)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10105 |
rorq(op1, 32); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10106 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10107 |
subl(offs, 2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10108 |
movq(sum, Address(out, offs, Address::times_4, 0)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10109 |
rorq(sum, 32); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10110 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10111 |
if (UseBMI2Instructions) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10112 |
multiply_add_64_bmi2(sum, op1, op2, carry, raxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10113 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10114 |
else { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10115 |
multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10116 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10117 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10118 |
// Store back in big endian from little endian |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10119 |
rorq(sum, 0x20); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10120 |
movq(Address(out, offs, Address::times_4, 0), sum); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10121 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10122 |
testl(len, len); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10123 |
jccb(Assembler::zero, L_carry); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10124 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10125 |
//Multiply the last in[] entry, if any |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10126 |
bind(L_last_in); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10127 |
movl(op1, Address(in, 0)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10128 |
movl(sum, Address(out, offs, Address::times_4, -4)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10129 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10130 |
movl(raxReg, k); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10131 |
mull(op1); //tmp4 * eax -> edx:eax |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10132 |
addl(sum, carry); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10133 |
adcl(rdxReg, 0); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10134 |
addl(sum, raxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10135 |
adcl(rdxReg, 0); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10136 |
movl(carry, rdxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10137 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10138 |
movl(Address(out, offs, Address::times_4, -4), sum); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10139 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10140 |
bind(L_carry); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10141 |
//return tmp5/carry as carry in rax |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10142 |
movl(rax, carry); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10143 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10144 |
bind(L_done); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10145 |
pop(tmp5); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10146 |
pop(tmp4); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10147 |
pop(tmp3); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10148 |
pop(tmp2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10149 |
pop(tmp1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
10150 |
} |
26434
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
10151 |
#endif |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
10152 |
|
18507
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10153 |
/** |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10154 |
* Emits code to update CRC-32 with a byte value according to constants in table |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10155 |
* |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10156 |
* @param [in,out]crc Register containing the crc. |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10157 |
* @param [in]val Register containing the byte to fold into the CRC. |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10158 |
* @param [in]table Register containing the table of crc constants. |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10159 |
* |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10160 |
* uint32_t crc; |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10161 |
* val = crc_table[(val ^ crc) & 0xFF]; |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10162 |
* crc = val ^ (crc >> 8); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10163 |
* |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10164 |
*/ |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10165 |
void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) { |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10166 |
xorl(val, crc); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10167 |
andl(val, 0xFF); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10168 |
shrl(crc, 8); // unsigned shift |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10169 |
xorl(crc, Address(table, val, Address::times_4, 0)); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10170 |
} |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10171 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10172 |
/** |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10173 |
* Fold 128-bit data chunk |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10174 |
*/ |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10175 |
void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) { |
25932
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10176 |
if (UseAVX > 0) { |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10177 |
vpclmulhdq(xtmp, xK, xcrc); // [123:64] |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10178 |
vpclmulldq(xcrc, xK, xcrc); // [63:0] |
30624 | 10179 |
vpxor(xcrc, xcrc, Address(buf, offset), 0 /* vector_len */); |
25932
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10180 |
pxor(xcrc, xtmp); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10181 |
} else { |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10182 |
movdqa(xtmp, xcrc); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10183 |
pclmulhdq(xtmp, xK); // [123:64] |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10184 |
pclmulldq(xcrc, xK); // [63:0] |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10185 |
pxor(xcrc, xtmp); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10186 |
movdqu(xtmp, Address(buf, offset)); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10187 |
pxor(xcrc, xtmp); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10188 |
} |
18507
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10189 |
} |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10190 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10191 |
void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf) { |
25932
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10192 |
if (UseAVX > 0) { |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10193 |
vpclmulhdq(xtmp, xK, xcrc); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10194 |
vpclmulldq(xcrc, xK, xcrc); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10195 |
pxor(xcrc, xbuf); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10196 |
pxor(xcrc, xtmp); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10197 |
} else { |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10198 |
movdqa(xtmp, xcrc); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10199 |
pclmulhdq(xtmp, xK); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10200 |
pclmulldq(xcrc, xK); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10201 |
pxor(xcrc, xbuf); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10202 |
pxor(xcrc, xtmp); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10203 |
} |
18507
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10204 |
} |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10205 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10206 |
/** |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10207 |
* 8-bit folds to compute 32-bit CRC |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10208 |
* |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10209 |
* uint64_t xcrc; |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10210 |
* timesXtoThe32[xcrc & 0xFF] ^ (xcrc >> 8); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10211 |
*/ |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10212 |
void MacroAssembler::fold_8bit_crc32(XMMRegister xcrc, Register table, XMMRegister xtmp, Register tmp) { |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10213 |
movdl(tmp, xcrc); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10214 |
andl(tmp, 0xFF); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10215 |
movdl(xtmp, Address(table, tmp, Address::times_4, 0)); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10216 |
psrldq(xcrc, 1); // unsigned shift one byte |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10217 |
pxor(xcrc, xtmp); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10218 |
} |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10219 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10220 |
/** |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10221 |
* uint32_t crc; |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10222 |
* timesXtoThe32[crc & 0xFF] ^ (crc >> 8); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10223 |
*/ |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10224 |
void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) { |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10225 |
movl(tmp, crc); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10226 |
andl(tmp, 0xFF); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10227 |
shrl(crc, 8); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10228 |
xorl(crc, Address(table, tmp, Address::times_4, 0)); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10229 |
} |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10230 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10231 |
/** |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10232 |
* @param crc register containing existing CRC (32-bit) |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10233 |
* @param buf register pointing to input byte buffer (byte*) |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10234 |
* @param len register containing number of bytes |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10235 |
* @param table register that will contain address of CRC table |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10236 |
* @param tmp scratch register |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10237 |
*/ |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10238 |
void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp) { |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10239 |
assert_different_registers(crc, buf, len, table, tmp, rax); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10240 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10241 |
Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned; |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10242 |
Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop; |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10243 |
|
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
10244 |
// For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
10245 |
// context for the registers used, where all instructions below are using 128-bit mode |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
10246 |
// On EVEX without VL and BW, these instructions will all be AVX. |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
10247 |
if (VM_Version::supports_avx512vlbw()) { |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
10248 |
movl(tmp, 0xffff); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
10249 |
kmovwl(k1, tmp); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
10250 |
} |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
10251 |
|
18507
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10252 |
lea(table, ExternalAddress(StubRoutines::crc_table_addr())); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10253 |
notl(crc); // ~crc |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10254 |
cmpl(len, 16); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10255 |
jcc(Assembler::less, L_tail); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10256 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10257 |
// Align buffer to 16 bytes |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10258 |
movl(tmp, buf); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10259 |
andl(tmp, 0xF); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10260 |
jccb(Assembler::zero, L_aligned); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10261 |
subl(tmp, 16); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10262 |
addl(len, tmp); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10263 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10264 |
align(4); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10265 |
BIND(L_align_loop); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10266 |
movsbl(rax, Address(buf, 0)); // load byte with sign extension |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10267 |
update_byte_crc32(crc, rax, table); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10268 |
increment(buf); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10269 |
incrementl(tmp); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10270 |
jccb(Assembler::less, L_align_loop); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10271 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10272 |
BIND(L_aligned); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10273 |
movl(tmp, len); // save |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10274 |
shrl(len, 4); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10275 |
jcc(Assembler::zero, L_tail_restore); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10276 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10277 |
// Fold crc into first bytes of vector |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10278 |
movdqa(xmm1, Address(buf, 0)); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10279 |
movdl(rax, xmm1); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10280 |
xorl(crc, rax); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10281 |
pinsrd(xmm1, crc, 0); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10282 |
addptr(buf, 16); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10283 |
subl(len, 4); // len > 0 |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10284 |
jcc(Assembler::less, L_fold_tail); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10285 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10286 |
movdqa(xmm2, Address(buf, 0)); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10287 |
movdqa(xmm3, Address(buf, 16)); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10288 |
movdqa(xmm4, Address(buf, 32)); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10289 |
addptr(buf, 48); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10290 |
subl(len, 3); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10291 |
jcc(Assembler::lessEqual, L_fold_512b); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10292 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10293 |
// Fold total 512 bits of polynomial on each iteration, |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10294 |
// 128 bits per each of 4 parallel streams. |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10295 |
movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32)); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10296 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10297 |
align(32); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10298 |
BIND(L_fold_512b_loop); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10299 |
fold_128bit_crc32(xmm1, xmm0, xmm5, buf, 0); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10300 |
fold_128bit_crc32(xmm2, xmm0, xmm5, buf, 16); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10301 |
fold_128bit_crc32(xmm3, xmm0, xmm5, buf, 32); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10302 |
fold_128bit_crc32(xmm4, xmm0, xmm5, buf, 48); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10303 |
addptr(buf, 64); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10304 |
subl(len, 4); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10305 |
jcc(Assembler::greater, L_fold_512b_loop); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10306 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10307 |
// Fold 512 bits to 128 bits. |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10308 |
BIND(L_fold_512b); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10309 |
movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16)); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10310 |
fold_128bit_crc32(xmm1, xmm0, xmm5, xmm2); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10311 |
fold_128bit_crc32(xmm1, xmm0, xmm5, xmm3); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10312 |
fold_128bit_crc32(xmm1, xmm0, xmm5, xmm4); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10313 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10314 |
// Fold the rest of 128 bits data chunks |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10315 |
BIND(L_fold_tail); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10316 |
addl(len, 3); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10317 |
jccb(Assembler::lessEqual, L_fold_128b); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10318 |
movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16)); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10319 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10320 |
BIND(L_fold_tail_loop); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10321 |
fold_128bit_crc32(xmm1, xmm0, xmm5, buf, 0); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10322 |
addptr(buf, 16); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10323 |
decrementl(len); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10324 |
jccb(Assembler::greater, L_fold_tail_loop); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10325 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10326 |
// Fold 128 bits in xmm1 down into 32 bits in crc register. |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10327 |
BIND(L_fold_128b); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10328 |
movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr())); |
25932
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10329 |
if (UseAVX > 0) { |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10330 |
vpclmulqdq(xmm2, xmm0, xmm1, 0x1); |
30624 | 10331 |
vpand(xmm3, xmm0, xmm2, 0 /* vector_len */); |
25932
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10332 |
vpclmulqdq(xmm0, xmm0, xmm3, 0x1); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10333 |
} else { |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10334 |
movdqa(xmm2, xmm0); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10335 |
pclmulqdq(xmm2, xmm1, 0x1); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10336 |
movdqa(xmm3, xmm0); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10337 |
pand(xmm3, xmm2); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10338 |
pclmulqdq(xmm0, xmm3, 0x1); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
10339 |
} |
18507
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10340 |
psrldq(xmm1, 8); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10341 |
psrldq(xmm2, 4); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10342 |
pxor(xmm0, xmm1); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10343 |
pxor(xmm0, xmm2); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10344 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10345 |
// 8 8-bit folds to compute 32-bit CRC. |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10346 |
for (int j = 0; j < 4; j++) { |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10347 |
fold_8bit_crc32(xmm0, table, xmm1, rax); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10348 |
} |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10349 |
movdl(crc, xmm0); // mov 32 bits to general register |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10350 |
for (int j = 0; j < 4; j++) { |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10351 |
fold_8bit_crc32(crc, table, rax); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10352 |
} |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10353 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10354 |
BIND(L_tail_restore); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10355 |
movl(len, tmp); // restore |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10356 |
BIND(L_tail); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10357 |
andl(len, 0xf); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10358 |
jccb(Assembler::zero, L_exit); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10359 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10360 |
// Fold the rest of bytes |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10361 |
align(4); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10362 |
BIND(L_tail_loop); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10363 |
movsbl(rax, Address(buf, 0)); // load byte with sign extension |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10364 |
update_byte_crc32(crc, rax, table); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10365 |
increment(buf); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10366 |
decrementl(len); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10367 |
jccb(Assembler::greater, L_tail_loop); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10368 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10369 |
BIND(L_exit); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10370 |
notl(crc); // ~c |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10371 |
} |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
10372 |
|
33066 | 10373 |
#ifdef _LP64 |
10374 |
// S. Gueron / Information Processing Letters 112 (2012) 184 |
|
10375 |
// Algorithm 4: Computing carry-less multiplication using a precomputed lookup table. |
|
10376 |
// Input: A 32 bit value B = [byte3, byte2, byte1, byte0]. |
|
10377 |
// Output: the 64-bit carry-less product of B * CONST |
|
10378 |
void MacroAssembler::crc32c_ipl_alg4(Register in, uint32_t n, |
|
10379 |
Register tmp1, Register tmp2, Register tmp3) { |
|
10380 |
lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr())); |
|
10381 |
if (n > 0) { |
|
10382 |
addq(tmp3, n * 256 * 8); |
|
10383 |
} |
|
10384 |
// Q1 = TABLEExt[n][B & 0xFF]; |
|
10385 |
movl(tmp1, in); |
|
10386 |
andl(tmp1, 0x000000FF); |
|
10387 |
shll(tmp1, 3); |
|
10388 |
addq(tmp1, tmp3); |
|
10389 |
movq(tmp1, Address(tmp1, 0)); |
|
10390 |
||
10391 |
// Q2 = TABLEExt[n][B >> 8 & 0xFF]; |
|
10392 |
movl(tmp2, in); |
|
10393 |
shrl(tmp2, 8); |
|
10394 |
andl(tmp2, 0x000000FF); |
|
10395 |
shll(tmp2, 3); |
|
10396 |
addq(tmp2, tmp3); |
|
10397 |
movq(tmp2, Address(tmp2, 0)); |
|
10398 |
||
10399 |
shlq(tmp2, 8); |
|
10400 |
xorq(tmp1, tmp2); |
|
10401 |
||
10402 |
// Q3 = TABLEExt[n][B >> 16 & 0xFF]; |
|
10403 |
movl(tmp2, in); |
|
10404 |
shrl(tmp2, 16); |
|
10405 |
andl(tmp2, 0x000000FF); |
|
10406 |
shll(tmp2, 3); |
|
10407 |
addq(tmp2, tmp3); |
|
10408 |
movq(tmp2, Address(tmp2, 0)); |
|
10409 |
||
10410 |
shlq(tmp2, 16); |
|
10411 |
xorq(tmp1, tmp2); |
|
10412 |
||
10413 |
// Q4 = TABLEExt[n][B >> 24 & 0xFF]; |
|
10414 |
shrl(in, 24); |
|
10415 |
andl(in, 0x000000FF); |
|
10416 |
shll(in, 3); |
|
10417 |
addq(in, tmp3); |
|
10418 |
movq(in, Address(in, 0)); |
|
10419 |
||
10420 |
shlq(in, 24); |
|
10421 |
xorq(in, tmp1); |
|
10422 |
// return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24; |
|
10423 |
} |
|
10424 |
||
10425 |
void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1, |
|
10426 |
Register in_out, |
|
10427 |
uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported, |
|
10428 |
XMMRegister w_xtmp2, |
|
10429 |
Register tmp1, |
|
10430 |
Register n_tmp2, Register n_tmp3) { |
|
10431 |
if (is_pclmulqdq_supported) { |
|
10432 |
movdl(w_xtmp1, in_out); // modified blindly |
|
10433 |
||
10434 |
movl(tmp1, const_or_pre_comp_const_index); |
|
10435 |
movdl(w_xtmp2, tmp1); |
|
10436 |
pclmulqdq(w_xtmp1, w_xtmp2, 0); |
|
10437 |
||
10438 |
movdq(in_out, w_xtmp1); |
|
10439 |
} else { |
|
10440 |
crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3); |
|
10441 |
} |
|
10442 |
} |
|
10443 |
||
10444 |
// Recombination Alternative 2: No bit-reflections |
|
10445 |
// T1 = (CRC_A * U1) << 1 |
|
10446 |
// T2 = (CRC_B * U2) << 1 |
|
10447 |
// C1 = T1 >> 32 |
|
10448 |
// C2 = T2 >> 32 |
|
10449 |
// T1 = T1 & 0xFFFFFFFF |
|
10450 |
// T2 = T2 & 0xFFFFFFFF |
|
10451 |
// T1 = CRC32(0, T1) |
|
10452 |
// T2 = CRC32(0, T2) |
|
10453 |
// C1 = C1 ^ T1 |
|
10454 |
// C2 = C2 ^ T2 |
|
10455 |
// CRC = C1 ^ C2 ^ CRC_C |
|
10456 |
void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2, |
|
10457 |
XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, |
|
10458 |
Register tmp1, Register tmp2, |
|
10459 |
Register n_tmp3) { |
|
10460 |
crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); |
|
10461 |
crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); |
|
10462 |
shlq(in_out, 1); |
|
10463 |
movl(tmp1, in_out); |
|
10464 |
shrq(in_out, 32); |
|
10465 |
xorl(tmp2, tmp2); |
|
10466 |
crc32(tmp2, tmp1, 4); |
|
10467 |
xorl(in_out, tmp2); // we don't care about upper 32 bit contents here |
|
10468 |
shlq(in1, 1); |
|
10469 |
movl(tmp1, in1); |
|
10470 |
shrq(in1, 32); |
|
10471 |
xorl(tmp2, tmp2); |
|
10472 |
crc32(tmp2, tmp1, 4); |
|
10473 |
xorl(in1, tmp2); |
|
10474 |
xorl(in_out, in1); |
|
10475 |
xorl(in_out, in2); |
|
10476 |
} |
|
10477 |
||
10478 |
// Set N to predefined value |
|
10479 |
// Subtract from a lenght of a buffer |
|
10480 |
// execute in a loop: |
|
10481 |
// CRC_A = 0xFFFFFFFF, CRC_B = 0, CRC_C = 0 |
|
10482 |
// for i = 1 to N do |
|
10483 |
// CRC_A = CRC32(CRC_A, A[i]) |
|
10484 |
// CRC_B = CRC32(CRC_B, B[i]) |
|
10485 |
// CRC_C = CRC32(CRC_C, C[i]) |
|
10486 |
// end for |
|
10487 |
// Recombine |
|
10488 |
void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, |
|
10489 |
Register in_out1, Register in_out2, Register in_out3, |
|
10490 |
Register tmp1, Register tmp2, Register tmp3, |
|
10491 |
XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, |
|
10492 |
Register tmp4, Register tmp5, |
|
10493 |
Register n_tmp6) { |
|
10494 |
Label L_processPartitions; |
|
10495 |
Label L_processPartition; |
|
10496 |
Label L_exit; |
|
10497 |
||
10498 |
bind(L_processPartitions); |
|
10499 |
cmpl(in_out1, 3 * size); |
|
10500 |
jcc(Assembler::less, L_exit); |
|
10501 |
xorl(tmp1, tmp1); |
|
10502 |
xorl(tmp2, tmp2); |
|
10503 |
movq(tmp3, in_out2); |
|
10504 |
addq(tmp3, size); |
|
10505 |
||
10506 |
bind(L_processPartition); |
|
10507 |
crc32(in_out3, Address(in_out2, 0), 8); |
|
10508 |
crc32(tmp1, Address(in_out2, size), 8); |
|
10509 |
crc32(tmp2, Address(in_out2, size * 2), 8); |
|
10510 |
addq(in_out2, 8); |
|
10511 |
cmpq(in_out2, tmp3); |
|
10512 |
jcc(Assembler::less, L_processPartition); |
|
10513 |
crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2, |
|
10514 |
w_xtmp1, w_xtmp2, w_xtmp3, |
|
10515 |
tmp4, tmp5, |
|
10516 |
n_tmp6); |
|
10517 |
addq(in_out2, 2 * size); |
|
10518 |
subl(in_out1, 3 * size); |
|
10519 |
jmp(L_processPartitions); |
|
10520 |
||
10521 |
bind(L_exit); |
|
10522 |
} |
|
10523 |
#else |
|
10524 |
void MacroAssembler::crc32c_ipl_alg4(Register in_out, uint32_t n, |
|
10525 |
Register tmp1, Register tmp2, Register tmp3, |
|
10526 |
XMMRegister xtmp1, XMMRegister xtmp2) { |
|
10527 |
lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr())); |
|
10528 |
if (n > 0) { |
|
10529 |
addl(tmp3, n * 256 * 8); |
|
10530 |
} |
|
10531 |
// Q1 = TABLEExt[n][B & 0xFF]; |
|
10532 |
movl(tmp1, in_out); |
|
10533 |
andl(tmp1, 0x000000FF); |
|
10534 |
shll(tmp1, 3); |
|
10535 |
addl(tmp1, tmp3); |
|
10536 |
movq(xtmp1, Address(tmp1, 0)); |
|
10537 |
||
10538 |
// Q2 = TABLEExt[n][B >> 8 & 0xFF]; |
|
10539 |
movl(tmp2, in_out); |
|
10540 |
shrl(tmp2, 8); |
|
10541 |
andl(tmp2, 0x000000FF); |
|
10542 |
shll(tmp2, 3); |
|
10543 |
addl(tmp2, tmp3); |
|
10544 |
movq(xtmp2, Address(tmp2, 0)); |
|
10545 |
||
10546 |
psllq(xtmp2, 8); |
|
10547 |
pxor(xtmp1, xtmp2); |
|
10548 |
||
10549 |
// Q3 = TABLEExt[n][B >> 16 & 0xFF]; |
|
10550 |
movl(tmp2, in_out); |
|
10551 |
shrl(tmp2, 16); |
|
10552 |
andl(tmp2, 0x000000FF); |
|
10553 |
shll(tmp2, 3); |
|
10554 |
addl(tmp2, tmp3); |
|
10555 |
movq(xtmp2, Address(tmp2, 0)); |
|
10556 |
||
10557 |
psllq(xtmp2, 16); |
|
10558 |
pxor(xtmp1, xtmp2); |
|
10559 |
||
10560 |
// Q4 = TABLEExt[n][B >> 24 & 0xFF]; |
|
10561 |
shrl(in_out, 24); |
|
10562 |
andl(in_out, 0x000000FF); |
|
10563 |
shll(in_out, 3); |
|
10564 |
addl(in_out, tmp3); |
|
10565 |
movq(xtmp2, Address(in_out, 0)); |
|
10566 |
||
10567 |
psllq(xtmp2, 24); |
|
10568 |
pxor(xtmp1, xtmp2); // Result in CXMM |
|
10569 |
// return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24; |
|
10570 |
} |
|
10571 |
||
10572 |
void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1, |
|
10573 |
Register in_out, |
|
10574 |
uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported, |
|
10575 |
XMMRegister w_xtmp2, |
|
10576 |
Register tmp1, |
|
10577 |
Register n_tmp2, Register n_tmp3) { |
|
10578 |
if (is_pclmulqdq_supported) { |
|
10579 |
movdl(w_xtmp1, in_out); |
|
10580 |
||
10581 |
movl(tmp1, const_or_pre_comp_const_index); |
|
10582 |
movdl(w_xtmp2, tmp1); |
|
10583 |
pclmulqdq(w_xtmp1, w_xtmp2, 0); |
|
10584 |
// Keep result in XMM since GPR is 32 bit in length |
|
10585 |
} else { |
|
10586 |
crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3, w_xtmp1, w_xtmp2); |
|
10587 |
} |
|
10588 |
} |
|
10589 |
||
10590 |
void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2, |
|
10591 |
XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, |
|
10592 |
Register tmp1, Register tmp2, |
|
10593 |
Register n_tmp3) { |
|
10594 |
crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); |
|
10595 |
crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); |
|
10596 |
||
10597 |
psllq(w_xtmp1, 1); |
|
10598 |
movdl(tmp1, w_xtmp1); |
|
10599 |
psrlq(w_xtmp1, 32); |
|
10600 |
movdl(in_out, w_xtmp1); |
|
10601 |
||
10602 |
xorl(tmp2, tmp2); |
|
10603 |
crc32(tmp2, tmp1, 4); |
|
10604 |
xorl(in_out, tmp2); |
|
10605 |
||
10606 |
psllq(w_xtmp2, 1); |
|
10607 |
movdl(tmp1, w_xtmp2); |
|
10608 |
psrlq(w_xtmp2, 32); |
|
10609 |
movdl(in1, w_xtmp2); |
|
10610 |
||
10611 |
xorl(tmp2, tmp2); |
|
10612 |
crc32(tmp2, tmp1, 4); |
|
10613 |
xorl(in1, tmp2); |
|
10614 |
xorl(in_out, in1); |
|
10615 |
xorl(in_out, in2); |
|
10616 |
} |
|
10617 |
||
10618 |
void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, |
|
10619 |
Register in_out1, Register in_out2, Register in_out3, |
|
10620 |
Register tmp1, Register tmp2, Register tmp3, |
|
10621 |
XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, |
|
10622 |
Register tmp4, Register tmp5, |
|
10623 |
Register n_tmp6) { |
|
10624 |
Label L_processPartitions; |
|
10625 |
Label L_processPartition; |
|
10626 |
Label L_exit; |
|
10627 |
||
10628 |
bind(L_processPartitions); |
|
10629 |
cmpl(in_out1, 3 * size); |
|
10630 |
jcc(Assembler::less, L_exit); |
|
10631 |
xorl(tmp1, tmp1); |
|
10632 |
xorl(tmp2, tmp2); |
|
10633 |
movl(tmp3, in_out2); |
|
10634 |
addl(tmp3, size); |
|
10635 |
||
10636 |
bind(L_processPartition); |
|
10637 |
crc32(in_out3, Address(in_out2, 0), 4); |
|
10638 |
crc32(tmp1, Address(in_out2, size), 4); |
|
10639 |
crc32(tmp2, Address(in_out2, size*2), 4); |
|
10640 |
crc32(in_out3, Address(in_out2, 0+4), 4); |
|
10641 |
crc32(tmp1, Address(in_out2, size+4), 4); |
|
10642 |
crc32(tmp2, Address(in_out2, size*2+4), 4); |
|
10643 |
addl(in_out2, 8); |
|
10644 |
cmpl(in_out2, tmp3); |
|
10645 |
jcc(Assembler::less, L_processPartition); |
|
10646 |
||
10647 |
push(tmp3); |
|
10648 |
push(in_out1); |
|
10649 |
push(in_out2); |
|
10650 |
tmp4 = tmp3; |
|
10651 |
tmp5 = in_out1; |
|
10652 |
n_tmp6 = in_out2; |
|
10653 |
||
10654 |
crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2, |
|
10655 |
w_xtmp1, w_xtmp2, w_xtmp3, |
|
10656 |
tmp4, tmp5, |
|
10657 |
n_tmp6); |
|
10658 |
||
10659 |
pop(in_out2); |
|
10660 |
pop(in_out1); |
|
10661 |
pop(tmp3); |
|
10662 |
||
10663 |
addl(in_out2, 2 * size); |
|
10664 |
subl(in_out1, 3 * size); |
|
10665 |
jmp(L_processPartitions); |
|
10666 |
||
10667 |
bind(L_exit); |
|
10668 |
} |
|
10669 |
#endif //LP64 |
|
10670 |
||
10671 |
#ifdef _LP64 |
|
10672 |
// Algorithm 2: Pipelined usage of the CRC32 instruction. |
|
10673 |
// Input: A buffer I of L bytes. |
|
10674 |
// Output: the CRC32C value of the buffer. |
|
10675 |
// Notations: |
|
10676 |
// Write L = 24N + r, with N = floor (L/24). |
|
10677 |
// r = L mod 24 (0 <= r < 24). |
|
10678 |
// Consider I as the concatenation of A|B|C|R, where A, B, C, each, |
|
10679 |
// N quadwords, and R consists of r bytes. |
|
10680 |
// A[j] = I [8j+7:8j], j= 0, 1, ..., N-1 |
|
10681 |
// B[j] = I [N + 8j+7:N + 8j], j= 0, 1, ..., N-1 |
|
10682 |
// C[j] = I [2N + 8j+7:2N + 8j], j= 0, 1, ..., N-1 |
|
10683 |
// if r > 0 R[j] = I [3N +j], j= 0, 1, ...,r-1 |
|
10684 |
void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2, |
|
10685 |
Register tmp1, Register tmp2, Register tmp3, |
|
10686 |
Register tmp4, Register tmp5, Register tmp6, |
|
10687 |
XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, |
|
10688 |
bool is_pclmulqdq_supported) { |
|
10689 |
uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS]; |
|
10690 |
Label L_wordByWord; |
|
10691 |
Label L_byteByByteProlog; |
|
10692 |
Label L_byteByByte; |
|
10693 |
Label L_exit; |
|
10694 |
||
10695 |
if (is_pclmulqdq_supported ) { |
|
10696 |
const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr; |
|
10697 |
const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr+1); |
|
10698 |
||
10699 |
const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2); |
|
10700 |
const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3); |
|
10701 |
||
10702 |
const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4); |
|
10703 |
const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5); |
|
10704 |
assert((CRC32C_NUM_PRECOMPUTED_CONSTANTS - 1 ) == 5, "Checking whether you declared all of the constants based on the number of \"chunks\""); |
|
10705 |
} else { |
|
10706 |
const_or_pre_comp_const_index[0] = 1; |
|
10707 |
const_or_pre_comp_const_index[1] = 0; |
|
10708 |
||
10709 |
const_or_pre_comp_const_index[2] = 3; |
|
10710 |
const_or_pre_comp_const_index[3] = 2; |
|
10711 |
||
10712 |
const_or_pre_comp_const_index[4] = 5; |
|
10713 |
const_or_pre_comp_const_index[5] = 4; |
|
10714 |
} |
|
10715 |
crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported, |
|
10716 |
in2, in1, in_out, |
|
10717 |
tmp1, tmp2, tmp3, |
|
10718 |
w_xtmp1, w_xtmp2, w_xtmp3, |
|
10719 |
tmp4, tmp5, |
|
10720 |
tmp6); |
|
10721 |
crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported, |
|
10722 |
in2, in1, in_out, |
|
10723 |
tmp1, tmp2, tmp3, |
|
10724 |
w_xtmp1, w_xtmp2, w_xtmp3, |
|
10725 |
tmp4, tmp5, |
|
10726 |
tmp6); |
|
10727 |
crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported, |
|
10728 |
in2, in1, in_out, |
|
10729 |
tmp1, tmp2, tmp3, |
|
10730 |
w_xtmp1, w_xtmp2, w_xtmp3, |
|
10731 |
tmp4, tmp5, |
|
10732 |
tmp6); |
|
10733 |
movl(tmp1, in2); |
|
10734 |
andl(tmp1, 0x00000007); |
|
10735 |
negl(tmp1); |
|
10736 |
addl(tmp1, in2); |
|
10737 |
addq(tmp1, in1); |
|
10738 |
||
10739 |
BIND(L_wordByWord); |
|
10740 |
cmpq(in1, tmp1); |
|
10741 |
jcc(Assembler::greaterEqual, L_byteByByteProlog); |
|
10742 |
crc32(in_out, Address(in1, 0), 4); |
|
10743 |
addq(in1, 4); |
|
10744 |
jmp(L_wordByWord); |
|
10745 |
||
10746 |
BIND(L_byteByByteProlog); |
|
10747 |
andl(in2, 0x00000007); |
|
10748 |
movl(tmp2, 1); |
|
10749 |
||
10750 |
BIND(L_byteByByte); |
|
10751 |
cmpl(tmp2, in2); |
|
10752 |
jccb(Assembler::greater, L_exit); |
|
10753 |
crc32(in_out, Address(in1, 0), 1); |
|
10754 |
incq(in1); |
|
10755 |
incl(tmp2); |
|
10756 |
jmp(L_byteByByte); |
|
10757 |
||
10758 |
BIND(L_exit); |
|
10759 |
} |
|
10760 |
#else |
|
10761 |
void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2, |
|
10762 |
Register tmp1, Register tmp2, Register tmp3, |
|
10763 |
Register tmp4, Register tmp5, Register tmp6, |
|
10764 |
XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, |
|
10765 |
bool is_pclmulqdq_supported) { |
|
10766 |
uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS]; |
|
10767 |
Label L_wordByWord; |
|
10768 |
Label L_byteByByteProlog; |
|
10769 |
Label L_byteByByte; |
|
10770 |
Label L_exit; |
|
10771 |
||
10772 |
if (is_pclmulqdq_supported) { |
|
10773 |
const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr; |
|
10774 |
const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 1); |
|
10775 |
||
10776 |
const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2); |
|
10777 |
const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3); |
|
10778 |
||
10779 |
const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4); |
|
10780 |
const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5); |
|
10781 |
} else { |
|
10782 |
const_or_pre_comp_const_index[0] = 1; |
|
10783 |
const_or_pre_comp_const_index[1] = 0; |
|
10784 |
||
10785 |
const_or_pre_comp_const_index[2] = 3; |
|
10786 |
const_or_pre_comp_const_index[3] = 2; |
|
10787 |
||
10788 |
const_or_pre_comp_const_index[4] = 5; |
|
10789 |
const_or_pre_comp_const_index[5] = 4; |
|
10790 |
} |
|
10791 |
crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported, |
|
10792 |
in2, in1, in_out, |
|
10793 |
tmp1, tmp2, tmp3, |
|
10794 |
w_xtmp1, w_xtmp2, w_xtmp3, |
|
10795 |
tmp4, tmp5, |
|
10796 |
tmp6); |
|
10797 |
crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported, |
|
10798 |
in2, in1, in_out, |
|
10799 |
tmp1, tmp2, tmp3, |
|
10800 |
w_xtmp1, w_xtmp2, w_xtmp3, |
|
10801 |
tmp4, tmp5, |
|
10802 |
tmp6); |
|
10803 |
crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported, |
|
10804 |
in2, in1, in_out, |
|
10805 |
tmp1, tmp2, tmp3, |
|
10806 |
w_xtmp1, w_xtmp2, w_xtmp3, |
|
10807 |
tmp4, tmp5, |
|
10808 |
tmp6); |
|
10809 |
movl(tmp1, in2); |
|
10810 |
andl(tmp1, 0x00000007); |
|
10811 |
negl(tmp1); |
|
10812 |
addl(tmp1, in2); |
|
10813 |
addl(tmp1, in1); |
|
10814 |
||
10815 |
BIND(L_wordByWord); |
|
10816 |
cmpl(in1, tmp1); |
|
10817 |
jcc(Assembler::greaterEqual, L_byteByByteProlog); |
|
10818 |
crc32(in_out, Address(in1,0), 4); |
|
10819 |
addl(in1, 4); |
|
10820 |
jmp(L_wordByWord); |
|
10821 |
||
10822 |
BIND(L_byteByByteProlog); |
|
10823 |
andl(in2, 0x00000007); |
|
10824 |
movl(tmp2, 1); |
|
10825 |
||
10826 |
BIND(L_byteByByte); |
|
10827 |
cmpl(tmp2, in2); |
|
10828 |
jccb(Assembler::greater, L_exit); |
|
10829 |
movb(tmp1, Address(in1, 0)); |
|
10830 |
crc32(in_out, tmp1, 1); |
|
10831 |
incl(in1); |
|
10832 |
incl(tmp2); |
|
10833 |
jmp(L_byteByByte); |
|
10834 |
||
10835 |
BIND(L_exit); |
|
10836 |
} |
|
10837 |
#endif // LP64 |
|
14626 | 10838 |
#undef BIND |
10839 |
#undef BLOCK_COMMENT |
|
10840 |
||
10841 |
||
33628 | 10842 |
// Compress char[] array to byte[]. |
10843 |
void MacroAssembler::char_array_compress(Register src, Register dst, Register len, |
|
10844 |
XMMRegister tmp1Reg, XMMRegister tmp2Reg, |
|
10845 |
XMMRegister tmp3Reg, XMMRegister tmp4Reg, |
|
10846 |
Register tmp5, Register result) { |
|
10847 |
Label copy_chars_loop, return_length, return_zero, done; |
|
10848 |
||
10849 |
// rsi: src |
|
10850 |
// rdi: dst |
|
10851 |
// rdx: len |
|
10852 |
// rcx: tmp5 |
|
10853 |
// rax: result |
|
10854 |
||
10855 |
// rsi holds start addr of source char[] to be compressed |
|
10856 |
// rdi holds start addr of destination byte[] |
|
10857 |
// rdx holds length |
|
10858 |
||
10859 |
assert(len != result, ""); |
|
10860 |
||
10861 |
// save length for return |
|
10862 |
push(len); |
|
10863 |
||
10864 |
if (UseSSE42Intrinsics) { |
|
34207
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
10865 |
assert(UseSSE >= 4, "SSE4 must be enabled for SSE4.2 intrinsics to be available"); |
33628 | 10866 |
Label copy_32_loop, copy_16, copy_tail; |
10867 |
||
10868 |
movl(result, len); |
|
10869 |
movl(tmp5, 0xff00ff00); // create mask to test for Unicode chars in vectors |
|
10870 |
||
10871 |
// vectored compression |
|
10872 |
andl(len, 0xfffffff0); // vector count (in chars) |
|
10873 |
andl(result, 0x0000000f); // tail count (in chars) |
|
10874 |
testl(len, len); |
|
10875 |
jccb(Assembler::zero, copy_16); |
|
10876 |
||
10877 |
// compress 16 chars per iter |
|
10878 |
movdl(tmp1Reg, tmp5); |
|
10879 |
pshufd(tmp1Reg, tmp1Reg, 0); // store Unicode mask in tmp1Reg |
|
10880 |
pxor(tmp4Reg, tmp4Reg); |
|
10881 |
||
10882 |
lea(src, Address(src, len, Address::times_2)); |
|
10883 |
lea(dst, Address(dst, len, Address::times_1)); |
|
10884 |
negptr(len); |
|
10885 |
||
10886 |
bind(copy_32_loop); |
|
10887 |
movdqu(tmp2Reg, Address(src, len, Address::times_2)); // load 1st 8 characters |
|
10888 |
por(tmp4Reg, tmp2Reg); |
|
10889 |
movdqu(tmp3Reg, Address(src, len, Address::times_2, 16)); // load next 8 characters |
|
10890 |
por(tmp4Reg, tmp3Reg); |
|
10891 |
ptest(tmp4Reg, tmp1Reg); // check for Unicode chars in next vector |
|
10892 |
jcc(Assembler::notZero, return_zero); |
|
10893 |
packuswb(tmp2Reg, tmp3Reg); // only ASCII chars; compress each to 1 byte |
|
10894 |
movdqu(Address(dst, len, Address::times_1), tmp2Reg); |
|
10895 |
addptr(len, 16); |
|
10896 |
jcc(Assembler::notZero, copy_32_loop); |
|
10897 |
||
10898 |
// compress next vector of 8 chars (if any) |
|
10899 |
bind(copy_16); |
|
10900 |
movl(len, result); |
|
10901 |
andl(len, 0xfffffff8); // vector count (in chars) |
|
10902 |
andl(result, 0x00000007); // tail count (in chars) |
|
10903 |
testl(len, len); |
|
10904 |
jccb(Assembler::zero, copy_tail); |
|
10905 |
||
10906 |
movdl(tmp1Reg, tmp5); |
|
10907 |
pshufd(tmp1Reg, tmp1Reg, 0); // store Unicode mask in tmp1Reg |
|
10908 |
pxor(tmp3Reg, tmp3Reg); |
|
10909 |
||
10910 |
movdqu(tmp2Reg, Address(src, 0)); |
|
10911 |
ptest(tmp2Reg, tmp1Reg); // check for Unicode chars in vector |
|
10912 |
jccb(Assembler::notZero, return_zero); |
|
10913 |
packuswb(tmp2Reg, tmp3Reg); // only LATIN1 chars; compress each to 1 byte |
|
10914 |
movq(Address(dst, 0), tmp2Reg); |
|
10915 |
addptr(src, 16); |
|
10916 |
addptr(dst, 8); |
|
10917 |
||
10918 |
bind(copy_tail); |
|
10919 |
movl(len, result); |
|
10920 |
} |
|
10921 |
// compress 1 char per iter |
|
10922 |
testl(len, len); |
|
10923 |
jccb(Assembler::zero, return_length); |
|
10924 |
lea(src, Address(src, len, Address::times_2)); |
|
10925 |
lea(dst, Address(dst, len, Address::times_1)); |
|
10926 |
negptr(len); |
|
10927 |
||
10928 |
bind(copy_chars_loop); |
|
10929 |
load_unsigned_short(result, Address(src, len, Address::times_2)); |
|
10930 |
testl(result, 0xff00); // check if Unicode char |
|
10931 |
jccb(Assembler::notZero, return_zero); |
|
10932 |
movb(Address(dst, len, Address::times_1), result); // ASCII char; compress to 1 byte |
|
10933 |
increment(len); |
|
10934 |
jcc(Assembler::notZero, copy_chars_loop); |
|
10935 |
||
10936 |
// if compression succeeded, return length |
|
10937 |
bind(return_length); |
|
10938 |
pop(result); |
|
10939 |
jmpb(done); |
|
10940 |
||
10941 |
// if compression failed, return 0 |
|
10942 |
bind(return_zero); |
|
10943 |
xorl(result, result); |
|
10944 |
addptr(rsp, wordSize); |
|
10945 |
||
10946 |
bind(done); |
|
10947 |
} |
|
10948 |
||
10949 |
// Inflate byte[] array to char[]. |
|
10950 |
void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len, |
|
10951 |
XMMRegister tmp1, Register tmp2) { |
|
10952 |
Label copy_chars_loop, done; |
|
10953 |
||
10954 |
// rsi: src |
|
10955 |
// rdi: dst |
|
10956 |
// rdx: len |
|
10957 |
// rcx: tmp2 |
|
10958 |
||
10959 |
// rsi holds start addr of source byte[] to be inflated |
|
10960 |
// rdi holds start addr of destination char[] |
|
10961 |
// rdx holds length |
|
10962 |
assert_different_registers(src, dst, len, tmp2); |
|
10963 |
||
10964 |
if (UseSSE42Intrinsics) { |
|
34207
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
10965 |
assert(UseSSE >= 4, "SSE4 must be enabled for SSE4.2 intrinsics to be available"); |
33628 | 10966 |
Label copy_8_loop, copy_bytes, copy_tail; |
10967 |
||
10968 |
movl(tmp2, len); |
|
10969 |
andl(tmp2, 0x00000007); // tail count (in chars) |
|
10970 |
andl(len, 0xfffffff8); // vector count (in chars) |
|
10971 |
jccb(Assembler::zero, copy_tail); |
|
10972 |
||
10973 |
// vectored inflation |
|
10974 |
lea(src, Address(src, len, Address::times_1)); |
|
10975 |
lea(dst, Address(dst, len, Address::times_2)); |
|
10976 |
negptr(len); |
|
10977 |
||
10978 |
// inflate 8 chars per iter |
|
10979 |
bind(copy_8_loop); |
|
10980 |
pmovzxbw(tmp1, Address(src, len, Address::times_1)); // unpack to 8 words |
|
10981 |
movdqu(Address(dst, len, Address::times_2), tmp1); |
|
10982 |
addptr(len, 8); |
|
10983 |
jcc(Assembler::notZero, copy_8_loop); |
|
10984 |
||
10985 |
bind(copy_tail); |
|
10986 |
movl(len, tmp2); |
|
10987 |
||
10988 |
cmpl(len, 4); |
|
10989 |
jccb(Assembler::less, copy_bytes); |
|
10990 |
||
10991 |
movdl(tmp1, Address(src, 0)); // load 4 byte chars |
|
10992 |
pmovzxbw(tmp1, tmp1); |
|
10993 |
movq(Address(dst, 0), tmp1); |
|
10994 |
subptr(len, 4); |
|
10995 |
addptr(src, 4); |
|
10996 |
addptr(dst, 8); |
|
10997 |
||
10998 |
bind(copy_bytes); |
|
10999 |
} |
|
11000 |
testl(len, len); |
|
11001 |
jccb(Assembler::zero, done); |
|
11002 |
lea(src, Address(src, len, Address::times_1)); |
|
11003 |
lea(dst, Address(dst, len, Address::times_2)); |
|
11004 |
negptr(len); |
|
11005 |
||
11006 |
// inflate 1 char per iter |
|
11007 |
bind(copy_chars_loop); |
|
11008 |
load_unsigned_byte(tmp2, Address(src, len, Address::times_1)); // load byte char |
|
11009 |
movw(Address(dst, len, Address::times_2), tmp2); // inflate byte char to word |
|
11010 |
increment(len); |
|
11011 |
jcc(Assembler::notZero, copy_chars_loop); |
|
11012 |
||
11013 |
bind(done); |
|
11014 |
} |
|
11015 |
||
11016 |
||
14626 | 11017 |
Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) { |
11018 |
switch (cond) { |
|
11019 |
// Note some conditions are synonyms for others |
|
11020 |
case Assembler::zero: return Assembler::notZero; |
|
11021 |
case Assembler::notZero: return Assembler::zero; |
|
11022 |
case Assembler::less: return Assembler::greaterEqual; |
|
11023 |
case Assembler::lessEqual: return Assembler::greater; |
|
11024 |
case Assembler::greater: return Assembler::lessEqual; |
|
11025 |
case Assembler::greaterEqual: return Assembler::less; |
|
11026 |
case Assembler::below: return Assembler::aboveEqual; |
|
11027 |
case Assembler::belowEqual: return Assembler::above; |
|
11028 |
case Assembler::above: return Assembler::belowEqual; |
|
11029 |
case Assembler::aboveEqual: return Assembler::below; |
|
11030 |
case Assembler::overflow: return Assembler::noOverflow; |
|
11031 |
case Assembler::noOverflow: return Assembler::overflow; |
|
11032 |
case Assembler::negative: return Assembler::positive; |
|
11033 |
case Assembler::positive: return Assembler::negative; |
|
11034 |
case Assembler::parity: return Assembler::noParity; |
|
11035 |
case Assembler::noParity: return Assembler::parity; |
|
11036 |
} |
|
11037 |
ShouldNotReachHere(); return Assembler::overflow; |
|
11038 |
} |
|
11039 |
||
11040 |
SkipIfEqual::SkipIfEqual( |
|
11041 |
MacroAssembler* masm, const bool* flag_addr, bool value) { |
|
11042 |
_masm = masm; |
|
11043 |
_masm->cmp8(ExternalAddress((address)flag_addr), value); |
|
11044 |
_masm->jcc(Assembler::equal, _label); |
|
11045 |
} |
|
11046 |
||
11047 |
SkipIfEqual::~SkipIfEqual() { |
|
11048 |
_masm->bind(_label); |
|
11049 |
} |
|
34633
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11050 |
|
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11051 |
// 32-bit Windows has its own fast-path implementation |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11052 |
// of get_thread |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11053 |
#if !defined(WIN32) || defined(_LP64) |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11054 |
|
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11055 |
// This is simply a call to Thread::current() |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11056 |
void MacroAssembler::get_thread(Register thread) { |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11057 |
if (thread != rax) { |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11058 |
push(rax); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11059 |
} |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11060 |
LP64_ONLY(push(rdi);) |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11061 |
LP64_ONLY(push(rsi);) |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11062 |
push(rdx); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11063 |
push(rcx); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11064 |
#ifdef _LP64 |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11065 |
push(r8); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11066 |
push(r9); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11067 |
push(r10); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11068 |
push(r11); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11069 |
#endif |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11070 |
|
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11071 |
MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11072 |
|
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11073 |
#ifdef _LP64 |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11074 |
pop(r11); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11075 |
pop(r10); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11076 |
pop(r9); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11077 |
pop(r8); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11078 |
#endif |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11079 |
pop(rcx); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11080 |
pop(rdx); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11081 |
LP64_ONLY(pop(rsi);) |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11082 |
LP64_ONLY(pop(rdi);) |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11083 |
if (thread != rax) { |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11084 |
mov(thread, rax); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11085 |
pop(rax); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11086 |
} |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11087 |
} |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11088 |
|
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
11089 |
#endif |