author | henryjen |
Mon, 14 Oct 2019 21:01:25 +0000 | |
changeset 58638 | 7be56b2ac50d |
parent 58625 | 9b54aee889b4 |
parent 58536 | 1b76d17440a0 |
child 58679 | 9c3209ff7550 |
child 58977 | c6a789f495fe |
permissions | -rw-r--r-- |
14626 | 1 |
/* |
57804 | 2 |
* Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved. |
14626 | 3 |
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
4 |
* |
|
5 |
* This code is free software; you can redistribute it and/or modify it |
|
6 |
* under the terms of the GNU General Public License version 2 only, as |
|
7 |
* published by the Free Software Foundation. |
|
8 |
* |
|
9 |
* This code is distributed in the hope that it will be useful, but WITHOUT |
|
10 |
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
|
11 |
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
|
12 |
* version 2 for more details (a copy is included in the LICENSE file that |
|
13 |
* accompanied this code). |
|
14 |
* |
|
15 |
* You should have received a copy of the GNU General Public License version |
|
16 |
* 2 along with this work; if not, write to the Free Software Foundation, |
|
17 |
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
|
18 |
* |
|
19 |
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
|
20 |
* or visit www.oracle.com if you need additional information or have any |
|
21 |
* questions. |
|
22 |
* |
|
23 |
*/ |
|
24 |
||
25 |
#include "precompiled.hpp" |
|
47765
b7c7428eaab9
8189610: Reconcile jvm.h and all jvm_md.h between java.base and hotspot
coleenp
parents:
47683
diff
changeset
|
26 |
#include "jvm.h" |
14626 | 27 |
#include "asm/assembler.hpp" |
28 |
#include "asm/assembler.inline.hpp" |
|
29 |
#include "compiler/disassembler.hpp" |
|
49748 | 30 |
#include "gc/shared/barrierSet.hpp" |
31 |
#include "gc/shared/barrierSetAssembler.hpp" |
|
30764 | 32 |
#include "gc/shared/collectedHeap.inline.hpp" |
14626 | 33 |
#include "interpreter/interpreter.hpp" |
34 |
#include "memory/resourceArea.hpp" |
|
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
35 |
#include "memory/universe.hpp" |
50446
39ca7558bc43
8203353: Fixup inferred decorators in the interpreter
eosterlund
parents:
50157
diff
changeset
|
36 |
#include "oops/accessDecorators.hpp" |
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
37 |
#include "oops/compressedOops.inline.hpp" |
31849
92ca49fa9fa7
8131344: Missing klass.inline.hpp include in compiler files
tschatzl
parents:
31782
diff
changeset
|
38 |
#include "oops/klass.inline.hpp" |
14626 | 39 |
#include "prims/methodHandles.hpp" |
40 |
#include "runtime/biasedLocking.hpp" |
|
49902
3661f31c6df4
8202150: [REDO] Split globals.hpp to factor out the Flag class
gziemski
parents:
49860
diff
changeset
|
41 |
#include "runtime/flags/flagSetting.hpp" |
49449
ef5d5d343e2a
8199263: Split interfaceSupport.hpp to not require including .inline.hpp files
coleenp
parents:
49347
diff
changeset
|
42 |
#include "runtime/interfaceSupport.inline.hpp" |
14626 | 43 |
#include "runtime/objectMonitor.hpp" |
44 |
#include "runtime/os.hpp" |
|
47881
0ce0ac68ace7
8189941: Implementation JEP 312: Thread-local handshake
rehn
parents:
47765
diff
changeset
|
45 |
#include "runtime/safepoint.hpp" |
0ce0ac68ace7
8189941: Implementation JEP 312: Thread-local handshake
rehn
parents:
47765
diff
changeset
|
46 |
#include "runtime/safepointMechanism.hpp" |
14626 | 47 |
#include "runtime/sharedRuntime.hpp" |
48 |
#include "runtime/stubRoutines.hpp" |
|
34633
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
49 |
#include "runtime/thread.hpp" |
15482
470d0b0c09f1
8005915: Unify SERIALGC and INCLUDE_ALTERNATE_GCS
jprovino
parents:
15117
diff
changeset
|
50 |
#include "utilities/macros.hpp" |
33066 | 51 |
#include "crc32c.h" |
33628 | 52 |
#ifdef COMPILER2 |
53 |
#include "opto/intrinsicnode.hpp" |
|
54 |
#endif |
|
14626 | 55 |
|
56 |
#ifdef PRODUCT |
|
57 |
#define BLOCK_COMMENT(str) /* nothing */ |
|
58 |
#define STOP(error) stop(error) |
|
59 |
#else |
|
60 |
#define BLOCK_COMMENT(str) block_comment(str) |
|
61 |
#define STOP(error) block_comment(error); stop(error) |
|
62 |
#endif |
|
63 |
||
64 |
#define BIND(label) bind(label); BLOCK_COMMENT(#label ":") |
|
65 |
||
14631
526804361522
8003250: SPARC: move MacroAssembler into separate file
twisti
parents:
14626
diff
changeset
|
66 |
#ifdef ASSERT |
526804361522
8003250: SPARC: move MacroAssembler into separate file
twisti
parents:
14626
diff
changeset
|
67 |
bool AbstractAssembler::pd_check_instruction_mark() { return true; } |
526804361522
8003250: SPARC: move MacroAssembler into separate file
twisti
parents:
14626
diff
changeset
|
68 |
#endif |
526804361522
8003250: SPARC: move MacroAssembler into separate file
twisti
parents:
14626
diff
changeset
|
69 |
|
14626 | 70 |
static Assembler::Condition reverse[] = { |
71 |
Assembler::noOverflow /* overflow = 0x0 */ , |
|
72 |
Assembler::overflow /* noOverflow = 0x1 */ , |
|
73 |
Assembler::aboveEqual /* carrySet = 0x2, below = 0x2 */ , |
|
74 |
Assembler::below /* aboveEqual = 0x3, carryClear = 0x3 */ , |
|
75 |
Assembler::notZero /* zero = 0x4, equal = 0x4 */ , |
|
76 |
Assembler::zero /* notZero = 0x5, notEqual = 0x5 */ , |
|
77 |
Assembler::above /* belowEqual = 0x6 */ , |
|
78 |
Assembler::belowEqual /* above = 0x7 */ , |
|
79 |
Assembler::positive /* negative = 0x8 */ , |
|
80 |
Assembler::negative /* positive = 0x9 */ , |
|
81 |
Assembler::noParity /* parity = 0xa */ , |
|
82 |
Assembler::parity /* noParity = 0xb */ , |
|
83 |
Assembler::greaterEqual /* less = 0xc */ , |
|
84 |
Assembler::less /* greaterEqual = 0xd */ , |
|
85 |
Assembler::greater /* lessEqual = 0xe */ , |
|
86 |
Assembler::lessEqual /* greater = 0xf, */ |
|
87 |
||
88 |
}; |
|
89 |
||
90 |
||
91 |
// Implementation of MacroAssembler |
|
92 |
||
93 |
// First all the versions that have distinct versions depending on 32/64 bit |
|
94 |
// Unless the difference is trivial (1 line or so). |
|
95 |
||
96 |
#ifndef _LP64 |
|
97 |
||
98 |
// 32bit versions |
|
99 |
||
100 |
Address MacroAssembler::as_Address(AddressLiteral adr) { |
|
101 |
return Address(adr.target(), adr.rspec()); |
|
102 |
} |
|
103 |
||
104 |
Address MacroAssembler::as_Address(ArrayAddress adr) { |
|
105 |
return Address::make_array(adr); |
|
106 |
} |
|
107 |
||
108 |
void MacroAssembler::call_VM_leaf_base(address entry_point, |
|
109 |
int number_of_arguments) { |
|
110 |
call(RuntimeAddress(entry_point)); |
|
111 |
increment(rsp, number_of_arguments * wordSize); |
|
112 |
} |
|
113 |
||
114 |
void MacroAssembler::cmpklass(Address src1, Metadata* obj) { |
|
115 |
cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate()); |
|
116 |
} |
|
117 |
||
118 |
void MacroAssembler::cmpklass(Register src1, Metadata* obj) { |
|
119 |
cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate()); |
|
120 |
} |
|
121 |
||
50536
8434981a4137
8203157: Object equals abstraction for BarrierSetAssembler
rkennke
parents:
50534
diff
changeset
|
122 |
void MacroAssembler::cmpoop_raw(Address src1, jobject obj) { |
8434981a4137
8203157: Object equals abstraction for BarrierSetAssembler
rkennke
parents:
50534
diff
changeset
|
123 |
cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate()); |
8434981a4137
8203157: Object equals abstraction for BarrierSetAssembler
rkennke
parents:
50534
diff
changeset
|
124 |
} |
8434981a4137
8203157: Object equals abstraction for BarrierSetAssembler
rkennke
parents:
50534
diff
changeset
|
125 |
|
8434981a4137
8203157: Object equals abstraction for BarrierSetAssembler
rkennke
parents:
50534
diff
changeset
|
126 |
void MacroAssembler::cmpoop_raw(Register src1, jobject obj) { |
8434981a4137
8203157: Object equals abstraction for BarrierSetAssembler
rkennke
parents:
50534
diff
changeset
|
127 |
cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate()); |
8434981a4137
8203157: Object equals abstraction for BarrierSetAssembler
rkennke
parents:
50534
diff
changeset
|
128 |
} |
8434981a4137
8203157: Object equals abstraction for BarrierSetAssembler
rkennke
parents:
50534
diff
changeset
|
129 |
|
14626 | 130 |
void MacroAssembler::cmpoop(Address src1, jobject obj) { |
50536
8434981a4137
8203157: Object equals abstraction for BarrierSetAssembler
rkennke
parents:
50534
diff
changeset
|
131 |
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); |
8434981a4137
8203157: Object equals abstraction for BarrierSetAssembler
rkennke
parents:
50534
diff
changeset
|
132 |
bs->obj_equals(this, src1, obj); |
14626 | 133 |
} |
134 |
||
135 |
void MacroAssembler::cmpoop(Register src1, jobject obj) { |
|
50536
8434981a4137
8203157: Object equals abstraction for BarrierSetAssembler
rkennke
parents:
50534
diff
changeset
|
136 |
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); |
8434981a4137
8203157: Object equals abstraction for BarrierSetAssembler
rkennke
parents:
50534
diff
changeset
|
137 |
bs->obj_equals(this, src1, obj); |
14626 | 138 |
} |
139 |
||
140 |
void MacroAssembler::extend_sign(Register hi, Register lo) { |
|
141 |
// According to Intel Doc. AP-526, "Integer Divide", p.18. |
|
142 |
if (VM_Version::is_P6() && hi == rdx && lo == rax) { |
|
143 |
cdql(); |
|
144 |
} else { |
|
145 |
movl(hi, lo); |
|
146 |
sarl(hi, 31); |
|
147 |
} |
|
148 |
} |
|
149 |
||
150 |
void MacroAssembler::jC2(Register tmp, Label& L) { |
|
151 |
// set parity bit if FPU flag C2 is set (via rax) |
|
152 |
save_rax(tmp); |
|
153 |
fwait(); fnstsw_ax(); |
|
154 |
sahf(); |
|
155 |
restore_rax(tmp); |
|
156 |
// branch |
|
157 |
jcc(Assembler::parity, L); |
|
158 |
} |
|
159 |
||
160 |
void MacroAssembler::jnC2(Register tmp, Label& L) { |
|
161 |
// set parity bit if FPU flag C2 is set (via rax) |
|
162 |
save_rax(tmp); |
|
163 |
fwait(); fnstsw_ax(); |
|
164 |
sahf(); |
|
165 |
restore_rax(tmp); |
|
166 |
// branch |
|
167 |
jcc(Assembler::noParity, L); |
|
168 |
} |
|
169 |
||
170 |
// 32bit can do a case table jump in one instruction but we no longer allow the base |
|
171 |
// to be installed in the Address class |
|
172 |
void MacroAssembler::jump(ArrayAddress entry) { |
|
173 |
jmp(as_Address(entry)); |
|
174 |
} |
|
175 |
||
176 |
// Note: y_lo will be destroyed |
|
177 |
void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) { |
|
178 |
// Long compare for Java (semantics as described in JVM spec.) |
|
179 |
Label high, low, done; |
|
180 |
||
181 |
cmpl(x_hi, y_hi); |
|
182 |
jcc(Assembler::less, low); |
|
183 |
jcc(Assembler::greater, high); |
|
184 |
// x_hi is the return register |
|
185 |
xorl(x_hi, x_hi); |
|
186 |
cmpl(x_lo, y_lo); |
|
187 |
jcc(Assembler::below, low); |
|
188 |
jcc(Assembler::equal, done); |
|
189 |
||
190 |
bind(high); |
|
191 |
xorl(x_hi, x_hi); |
|
192 |
increment(x_hi); |
|
193 |
jmp(done); |
|
194 |
||
195 |
bind(low); |
|
196 |
xorl(x_hi, x_hi); |
|
197 |
decrementl(x_hi); |
|
198 |
||
199 |
bind(done); |
|
200 |
} |
|
201 |
||
202 |
void MacroAssembler::lea(Register dst, AddressLiteral src) { |
|
203 |
mov_literal32(dst, (int32_t)src.target(), src.rspec()); |
|
204 |
} |
|
205 |
||
206 |
void MacroAssembler::lea(Address dst, AddressLiteral adr) { |
|
207 |
// leal(dst, as_Address(adr)); |
|
208 |
// see note in movl as to why we must use a move |
|
209 |
mov_literal32(dst, (int32_t) adr.target(), adr.rspec()); |
|
210 |
} |
|
211 |
||
212 |
void MacroAssembler::leave() { |
|
213 |
mov(rsp, rbp); |
|
214 |
pop(rbp); |
|
215 |
} |
|
216 |
||
217 |
void MacroAssembler::lmul(int x_rsp_offset, int y_rsp_offset) { |
|
218 |
// Multiplication of two Java long values stored on the stack |
|
219 |
// as illustrated below. Result is in rdx:rax. |
|
220 |
// |
|
221 |
// rsp ---> [ ?? ] \ \ |
|
222 |
// .... | y_rsp_offset | |
|
223 |
// [ y_lo ] / (in bytes) | x_rsp_offset |
|
224 |
// [ y_hi ] | (in bytes) |
|
225 |
// .... | |
|
226 |
// [ x_lo ] / |
|
227 |
// [ x_hi ] |
|
228 |
// .... |
|
229 |
// |
|
230 |
// Basic idea: lo(result) = lo(x_lo * y_lo) |
|
231 |
// hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi) |
|
232 |
Address x_hi(rsp, x_rsp_offset + wordSize); Address x_lo(rsp, x_rsp_offset); |
|
233 |
Address y_hi(rsp, y_rsp_offset + wordSize); Address y_lo(rsp, y_rsp_offset); |
|
234 |
Label quick; |
|
235 |
// load x_hi, y_hi and check if quick |
|
236 |
// multiplication is possible |
|
237 |
movl(rbx, x_hi); |
|
238 |
movl(rcx, y_hi); |
|
239 |
movl(rax, rbx); |
|
240 |
orl(rbx, rcx); // rbx, = 0 <=> x_hi = 0 and y_hi = 0 |
|
241 |
jcc(Assembler::zero, quick); // if rbx, = 0 do quick multiply |
|
242 |
// do full multiplication |
|
243 |
// 1st step |
|
244 |
mull(y_lo); // x_hi * y_lo |
|
245 |
movl(rbx, rax); // save lo(x_hi * y_lo) in rbx, |
|
246 |
// 2nd step |
|
247 |
movl(rax, x_lo); |
|
248 |
mull(rcx); // x_lo * y_hi |
|
249 |
addl(rbx, rax); // add lo(x_lo * y_hi) to rbx, |
|
250 |
// 3rd step |
|
251 |
bind(quick); // note: rbx, = 0 if quick multiply! |
|
252 |
movl(rax, x_lo); |
|
253 |
mull(y_lo); // x_lo * y_lo |
|
254 |
addl(rdx, rbx); // correct hi(x_lo * y_lo) |
|
255 |
} |
|
256 |
||
257 |
void MacroAssembler::lneg(Register hi, Register lo) { |
|
258 |
negl(lo); |
|
259 |
adcl(hi, 0); |
|
260 |
negl(hi); |
|
261 |
} |
|
262 |
||
263 |
void MacroAssembler::lshl(Register hi, Register lo) { |
|
264 |
// Java shift left long support (semantics as described in JVM spec., p.305) |
|
265 |
// (basic idea for shift counts s >= n: x << s == (x << n) << (s - n)) |
|
266 |
// shift value is in rcx ! |
|
267 |
assert(hi != rcx, "must not use rcx"); |
|
268 |
assert(lo != rcx, "must not use rcx"); |
|
269 |
const Register s = rcx; // shift count |
|
270 |
const int n = BitsPerWord; |
|
271 |
Label L; |
|
272 |
andl(s, 0x3f); // s := s & 0x3f (s < 0x40) |
|
273 |
cmpl(s, n); // if (s < n) |
|
274 |
jcc(Assembler::less, L); // else (s >= n) |
|
275 |
movl(hi, lo); // x := x << n |
|
276 |
xorl(lo, lo); |
|
277 |
// Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n! |
|
278 |
bind(L); // s (mod n) < n |
|
279 |
shldl(hi, lo); // x := x << s |
|
280 |
shll(lo); |
|
281 |
} |
|
282 |
||
283 |
||
284 |
void MacroAssembler::lshr(Register hi, Register lo, bool sign_extension) { |
|
285 |
// Java shift right long support (semantics as described in JVM spec., p.306 & p.310) |
|
286 |
// (basic idea for shift counts s >= n: x >> s == (x >> n) >> (s - n)) |
|
287 |
assert(hi != rcx, "must not use rcx"); |
|
288 |
assert(lo != rcx, "must not use rcx"); |
|
289 |
const Register s = rcx; // shift count |
|
290 |
const int n = BitsPerWord; |
|
291 |
Label L; |
|
292 |
andl(s, 0x3f); // s := s & 0x3f (s < 0x40) |
|
293 |
cmpl(s, n); // if (s < n) |
|
294 |
jcc(Assembler::less, L); // else (s >= n) |
|
295 |
movl(lo, hi); // x := x >> n |
|
296 |
if (sign_extension) sarl(hi, 31); |
|
297 |
else xorl(hi, hi); |
|
298 |
// Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n! |
|
299 |
bind(L); // s (mod n) < n |
|
300 |
shrdl(lo, hi); // x := x >> s |
|
301 |
if (sign_extension) sarl(hi); |
|
302 |
else shrl(hi); |
|
303 |
} |
|
304 |
||
305 |
void MacroAssembler::movoop(Register dst, jobject obj) { |
|
306 |
mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate()); |
|
307 |
} |
|
308 |
||
309 |
void MacroAssembler::movoop(Address dst, jobject obj) { |
|
310 |
mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate()); |
|
311 |
} |
|
312 |
||
313 |
void MacroAssembler::mov_metadata(Register dst, Metadata* obj) { |
|
314 |
mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate()); |
|
315 |
} |
|
316 |
||
317 |
void MacroAssembler::mov_metadata(Address dst, Metadata* obj) { |
|
318 |
mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate()); |
|
319 |
} |
|
320 |
||
23491 | 321 |
void MacroAssembler::movptr(Register dst, AddressLiteral src, Register scratch) { |
322 |
// scratch register is not used, |
|
323 |
// it is defined to match parameters of 64-bit version of this method. |
|
14626 | 324 |
if (src.is_lval()) { |
325 |
mov_literal32(dst, (intptr_t)src.target(), src.rspec()); |
|
326 |
} else { |
|
327 |
movl(dst, as_Address(src)); |
|
328 |
} |
|
329 |
} |
|
330 |
||
331 |
void MacroAssembler::movptr(ArrayAddress dst, Register src) { |
|
332 |
movl(as_Address(dst), src); |
|
333 |
} |
|
334 |
||
335 |
void MacroAssembler::movptr(Register dst, ArrayAddress src) { |
|
336 |
movl(dst, as_Address(src)); |
|
337 |
} |
|
338 |
||
339 |
// src should NEVER be a real pointer. Use AddressLiteral for true pointers |
|
340 |
void MacroAssembler::movptr(Address dst, intptr_t src) { |
|
341 |
movl(dst, src); |
|
342 |
} |
|
343 |
||
344 |
||
345 |
void MacroAssembler::pop_callee_saved_registers() { |
|
346 |
pop(rcx); |
|
347 |
pop(rdx); |
|
348 |
pop(rdi); |
|
349 |
pop(rsi); |
|
350 |
} |
|
351 |
||
352 |
void MacroAssembler::pop_fTOS() { |
|
353 |
fld_d(Address(rsp, 0)); |
|
354 |
addl(rsp, 2 * wordSize); |
|
355 |
} |
|
356 |
||
357 |
void MacroAssembler::push_callee_saved_registers() { |
|
358 |
push(rsi); |
|
359 |
push(rdi); |
|
360 |
push(rdx); |
|
361 |
push(rcx); |
|
362 |
} |
|
363 |
||
364 |
void MacroAssembler::push_fTOS() { |
|
365 |
subl(rsp, 2 * wordSize); |
|
366 |
fstp_d(Address(rsp, 0)); |
|
367 |
} |
|
368 |
||
369 |
||
370 |
void MacroAssembler::pushoop(jobject obj) { |
|
371 |
push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate()); |
|
372 |
} |
|
373 |
||
374 |
void MacroAssembler::pushklass(Metadata* obj) { |
|
375 |
push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate()); |
|
376 |
} |
|
377 |
||
378 |
void MacroAssembler::pushptr(AddressLiteral src) { |
|
379 |
if (src.is_lval()) { |
|
380 |
push_literal32((int32_t)src.target(), src.rspec()); |
|
381 |
} else { |
|
382 |
pushl(as_Address(src)); |
|
383 |
} |
|
384 |
} |
|
385 |
||
386 |
void MacroAssembler::set_word_if_not_zero(Register dst) { |
|
387 |
xorl(dst, dst); |
|
388 |
set_byte_if_not_zero(dst); |
|
389 |
} |
|
390 |
||
391 |
static void pass_arg0(MacroAssembler* masm, Register arg) { |
|
392 |
masm->push(arg); |
|
393 |
} |
|
394 |
||
395 |
static void pass_arg1(MacroAssembler* masm, Register arg) { |
|
396 |
masm->push(arg); |
|
397 |
} |
|
398 |
||
399 |
static void pass_arg2(MacroAssembler* masm, Register arg) { |
|
400 |
masm->push(arg); |
|
401 |
} |
|
402 |
||
403 |
static void pass_arg3(MacroAssembler* masm, Register arg) { |
|
404 |
masm->push(arg); |
|
405 |
} |
|
406 |
||
407 |
#ifndef PRODUCT |
|
408 |
extern "C" void findpc(intptr_t x); |
|
409 |
#endif |
|
410 |
||
411 |
void MacroAssembler::debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg) { |
|
412 |
// In order to get locks to work, we need to fake a in_VM state |
|
413 |
JavaThread* thread = JavaThread::current(); |
|
414 |
JavaThreadState saved_state = thread->thread_state(); |
|
415 |
thread->set_thread_state(_thread_in_vm); |
|
416 |
if (ShowMessageBoxOnError) { |
|
417 |
JavaThread* thread = JavaThread::current(); |
|
418 |
JavaThreadState saved_state = thread->thread_state(); |
|
419 |
thread->set_thread_state(_thread_in_vm); |
|
420 |
if (CountBytecodes || TraceBytecodes || StopInterpreterAt) { |
|
421 |
ttyLocker ttyl; |
|
422 |
BytecodeCounter::print(); |
|
423 |
} |
|
424 |
// To see where a verify_oop failed, get $ebx+40/X for this frame. |
|
425 |
// This is the value of eip which points to where verify_oop will return. |
|
426 |
if (os::message_box(msg, "Execution stopped, print registers?")) { |
|
427 |
print_state32(rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax, eip); |
|
428 |
BREAKPOINT; |
|
429 |
} |
|
58103
689a80d20550
8230762: Change MacroAssembler::debug32/64 to use fatal instead of assert
chagedorn
parents:
57893
diff
changeset
|
430 |
} |
689a80d20550
8230762: Change MacroAssembler::debug32/64 to use fatal instead of assert
chagedorn
parents:
57893
diff
changeset
|
431 |
fatal("DEBUG MESSAGE: %s", msg); |
14626 | 432 |
} |
433 |
||
434 |
void MacroAssembler::print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip) { |
|
435 |
ttyLocker ttyl; |
|
436 |
FlagSetting fs(Debugging, true); |
|
437 |
tty->print_cr("eip = 0x%08x", eip); |
|
438 |
#ifndef PRODUCT |
|
439 |
if ((WizardMode || Verbose) && PrintMiscellaneous) { |
|
440 |
tty->cr(); |
|
441 |
findpc(eip); |
|
442 |
tty->cr(); |
|
443 |
} |
|
444 |
#endif |
|
445 |
#define PRINT_REG(rax) \ |
|
446 |
{ tty->print("%s = ", #rax); os::print_location(tty, rax); } |
|
447 |
PRINT_REG(rax); |
|
448 |
PRINT_REG(rbx); |
|
449 |
PRINT_REG(rcx); |
|
450 |
PRINT_REG(rdx); |
|
451 |
PRINT_REG(rdi); |
|
452 |
PRINT_REG(rsi); |
|
453 |
PRINT_REG(rbp); |
|
454 |
PRINT_REG(rsp); |
|
455 |
#undef PRINT_REG |
|
456 |
// Print some words near top of staack. |
|
457 |
int* dump_sp = (int*) rsp; |
|
458 |
for (int col1 = 0; col1 < 8; col1++) { |
|
459 |
tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); |
|
460 |
os::print_location(tty, *dump_sp++); |
|
461 |
} |
|
462 |
for (int row = 0; row < 16; row++) { |
|
463 |
tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); |
|
464 |
for (int col = 0; col < 8; col++) { |
|
465 |
tty->print(" 0x%08x", *dump_sp++); |
|
466 |
} |
|
467 |
tty->cr(); |
|
468 |
} |
|
469 |
// Print some instructions around pc: |
|
470 |
Disassembler::decode((address)eip-64, (address)eip); |
|
471 |
tty->print_cr("--------"); |
|
472 |
Disassembler::decode((address)eip, (address)eip+32); |
|
473 |
} |
|
474 |
||
475 |
void MacroAssembler::stop(const char* msg) { |
|
476 |
ExternalAddress message((address)msg); |
|
477 |
// push address of message |
|
478 |
pushptr(message.addr()); |
|
479 |
{ Label L; call(L, relocInfo::none); bind(L); } // push eip |
|
480 |
pusha(); // push registers |
|
481 |
call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32))); |
|
482 |
hlt(); |
|
483 |
} |
|
484 |
||
485 |
void MacroAssembler::warn(const char* msg) { |
|
486 |
push_CPU_state(); |
|
487 |
||
488 |
ExternalAddress message((address) msg); |
|
489 |
// push address of message |
|
490 |
pushptr(message.addr()); |
|
491 |
||
492 |
call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning))); |
|
493 |
addl(rsp, wordSize); // discard argument |
|
494 |
pop_CPU_state(); |
|
495 |
} |
|
496 |
||
497 |
void MacroAssembler::print_state() { |
|
498 |
{ Label L; call(L, relocInfo::none); bind(L); } // push eip |
|
499 |
pusha(); // push registers |
|
500 |
||
501 |
push_CPU_state(); |
|
502 |
call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::print_state32))); |
|
503 |
pop_CPU_state(); |
|
504 |
||
505 |
popa(); |
|
506 |
addl(rsp, wordSize); |
|
507 |
} |
|
508 |
||
509 |
#else // _LP64 |
|
510 |
||
511 |
// 64 bit versions |
|
512 |
||
513 |
Address MacroAssembler::as_Address(AddressLiteral adr) { |
|
514 |
// amd64 always does this as a pc-rel |
|
515 |
// we can be absolute or disp based on the instruction type |
|
516 |
// jmp/call are displacements others are absolute |
|
517 |
assert(!adr.is_lval(), "must be rval"); |
|
518 |
assert(reachable(adr), "must be"); |
|
519 |
return Address((int32_t)(intptr_t)(adr.target() - pc()), adr.target(), adr.reloc()); |
|
520 |
||
521 |
} |
|
522 |
||
523 |
Address MacroAssembler::as_Address(ArrayAddress adr) { |
|
524 |
AddressLiteral base = adr.base(); |
|
525 |
lea(rscratch1, base); |
|
526 |
Address index = adr.index(); |
|
527 |
assert(index._disp == 0, "must not have disp"); // maybe it can? |
|
528 |
Address array(rscratch1, index._index, index._scale, index._disp); |
|
529 |
return array; |
|
530 |
} |
|
531 |
||
532 |
void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) { |
|
533 |
Label L, E; |
|
534 |
||
535 |
#ifdef _WIN64 |
|
536 |
// Windows always allocates space for it's register args |
|
537 |
assert(num_args <= 4, "only register arguments supported"); |
|
538 |
subq(rsp, frame::arg_reg_save_area_bytes); |
|
539 |
#endif |
|
540 |
||
541 |
// Align stack if necessary |
|
542 |
testl(rsp, 15); |
|
543 |
jcc(Assembler::zero, L); |
|
544 |
||
545 |
subq(rsp, 8); |
|
546 |
{ |
|
547 |
call(RuntimeAddress(entry_point)); |
|
548 |
} |
|
549 |
addq(rsp, 8); |
|
550 |
jmp(E); |
|
551 |
||
552 |
bind(L); |
|
553 |
{ |
|
554 |
call(RuntimeAddress(entry_point)); |
|
555 |
} |
|
556 |
||
557 |
bind(E); |
|
558 |
||
559 |
#ifdef _WIN64 |
|
560 |
// restore stack pointer |
|
561 |
addq(rsp, frame::arg_reg_save_area_bytes); |
|
562 |
#endif |
|
563 |
||
564 |
} |
|
565 |
||
566 |
void MacroAssembler::cmp64(Register src1, AddressLiteral src2) { |
|
567 |
assert(!src2.is_lval(), "should use cmpptr"); |
|
568 |
||
569 |
if (reachable(src2)) { |
|
570 |
cmpq(src1, as_Address(src2)); |
|
571 |
} else { |
|
572 |
lea(rscratch1, src2); |
|
573 |
Assembler::cmpq(src1, Address(rscratch1, 0)); |
|
574 |
} |
|
575 |
} |
|
576 |
||
577 |
int MacroAssembler::corrected_idivq(Register reg) { |
|
578 |
// Full implementation of Java ldiv and lrem; checks for special |
|
579 |
// case as described in JVM spec., p.243 & p.271. The function |
|
580 |
// returns the (pc) offset of the idivl instruction - may be needed |
|
581 |
// for implicit exceptions. |
|
582 |
// |
|
583 |
// normal case special case |
|
584 |
// |
|
585 |
// input : rax: dividend min_long |
|
586 |
// reg: divisor (may not be eax/edx) -1 |
|
587 |
// |
|
588 |
// output: rax: quotient (= rax idiv reg) min_long |
|
589 |
// rdx: remainder (= rax irem reg) 0 |
|
590 |
assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register"); |
|
591 |
static const int64_t min_long = 0x8000000000000000; |
|
592 |
Label normal_case, special_case; |
|
593 |
||
594 |
// check for special case |
|
595 |
cmp64(rax, ExternalAddress((address) &min_long)); |
|
596 |
jcc(Assembler::notEqual, normal_case); |
|
597 |
xorl(rdx, rdx); // prepare rdx for possible special case (where |
|
598 |
// remainder = 0) |
|
599 |
cmpq(reg, -1); |
|
600 |
jcc(Assembler::equal, special_case); |
|
601 |
||
602 |
// handle normal case |
|
603 |
bind(normal_case); |
|
604 |
cdqq(); |
|
605 |
int idivq_offset = offset(); |
|
606 |
idivq(reg); |
|
607 |
||
608 |
// normal and special case exit |
|
609 |
bind(special_case); |
|
610 |
||
611 |
return idivq_offset; |
|
612 |
} |
|
613 |
||
614 |
void MacroAssembler::decrementq(Register reg, int value) { |
|
615 |
if (value == min_jint) { subq(reg, value); return; } |
|
616 |
if (value < 0) { incrementq(reg, -value); return; } |
|
617 |
if (value == 0) { ; return; } |
|
618 |
if (value == 1 && UseIncDec) { decq(reg) ; return; } |
|
619 |
/* else */ { subq(reg, value) ; return; } |
|
620 |
} |
|
621 |
||
622 |
void MacroAssembler::decrementq(Address dst, int value) { |
|
623 |
if (value == min_jint) { subq(dst, value); return; } |
|
624 |
if (value < 0) { incrementq(dst, -value); return; } |
|
625 |
if (value == 0) { ; return; } |
|
626 |
if (value == 1 && UseIncDec) { decq(dst) ; return; } |
|
627 |
/* else */ { subq(dst, value) ; return; } |
|
628 |
} |
|
629 |
||
23491 | 630 |
void MacroAssembler::incrementq(AddressLiteral dst) { |
631 |
if (reachable(dst)) { |
|
632 |
incrementq(as_Address(dst)); |
|
633 |
} else { |
|
634 |
lea(rscratch1, dst); |
|
635 |
incrementq(Address(rscratch1, 0)); |
|
636 |
} |
|
637 |
} |
|
638 |
||
14626 | 639 |
void MacroAssembler::incrementq(Register reg, int value) { |
640 |
if (value == min_jint) { addq(reg, value); return; } |
|
641 |
if (value < 0) { decrementq(reg, -value); return; } |
|
642 |
if (value == 0) { ; return; } |
|
643 |
if (value == 1 && UseIncDec) { incq(reg) ; return; } |
|
644 |
/* else */ { addq(reg, value) ; return; } |
|
645 |
} |
|
646 |
||
647 |
void MacroAssembler::incrementq(Address dst, int value) { |
|
648 |
if (value == min_jint) { addq(dst, value); return; } |
|
649 |
if (value < 0) { decrementq(dst, -value); return; } |
|
650 |
if (value == 0) { ; return; } |
|
651 |
if (value == 1 && UseIncDec) { incq(dst) ; return; } |
|
652 |
/* else */ { addq(dst, value) ; return; } |
|
653 |
} |
|
654 |
||
655 |
// 32bit can do a case table jump in one instruction but we no longer allow the base |
|
656 |
// to be installed in the Address class |
|
657 |
void MacroAssembler::jump(ArrayAddress entry) { |
|
658 |
lea(rscratch1, entry.base()); |
|
659 |
Address dispatch = entry.index(); |
|
660 |
assert(dispatch._base == noreg, "must be"); |
|
661 |
dispatch._base = rscratch1; |
|
662 |
jmp(dispatch); |
|
663 |
} |
|
664 |
||
665 |
void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) { |
|
666 |
ShouldNotReachHere(); // 64bit doesn't use two regs |
|
667 |
cmpq(x_lo, y_lo); |
|
668 |
} |
|
669 |
||
670 |
void MacroAssembler::lea(Register dst, AddressLiteral src) { |
|
671 |
mov_literal64(dst, (intptr_t)src.target(), src.rspec()); |
|
672 |
} |
|
673 |
||
674 |
void MacroAssembler::lea(Address dst, AddressLiteral adr) { |
|
675 |
mov_literal64(rscratch1, (intptr_t)adr.target(), adr.rspec()); |
|
676 |
movptr(dst, rscratch1); |
|
677 |
} |
|
678 |
||
679 |
void MacroAssembler::leave() { |
|
680 |
// %%% is this really better? Why not on 32bit too? |
|
14837
a75c3082d106
8004250: replace AbstractAssembler a_byte/a_long with emit_int8/emit_int32
twisti
parents:
14834
diff
changeset
|
681 |
emit_int8((unsigned char)0xC9); // LEAVE |
14626 | 682 |
} |
683 |
||
684 |
void MacroAssembler::lneg(Register hi, Register lo) { |
|
685 |
ShouldNotReachHere(); // 64bit doesn't use two regs |
|
686 |
negq(lo); |
|
687 |
} |
|
688 |
||
689 |
void MacroAssembler::movoop(Register dst, jobject obj) { |
|
690 |
mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate()); |
|
691 |
} |
|
692 |
||
693 |
void MacroAssembler::movoop(Address dst, jobject obj) { |
|
694 |
mov_literal64(rscratch1, (intptr_t)obj, oop_Relocation::spec_for_immediate()); |
|
695 |
movq(dst, rscratch1); |
|
696 |
} |
|
697 |
||
698 |
void MacroAssembler::mov_metadata(Register dst, Metadata* obj) { |
|
699 |
mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate()); |
|
700 |
} |
|
701 |
||
702 |
void MacroAssembler::mov_metadata(Address dst, Metadata* obj) { |
|
703 |
mov_literal64(rscratch1, (intptr_t)obj, metadata_Relocation::spec_for_immediate()); |
|
704 |
movq(dst, rscratch1); |
|
705 |
} |
|
706 |
||
23491 | 707 |
void MacroAssembler::movptr(Register dst, AddressLiteral src, Register scratch) { |
14626 | 708 |
if (src.is_lval()) { |
709 |
mov_literal64(dst, (intptr_t)src.target(), src.rspec()); |
|
710 |
} else { |
|
711 |
if (reachable(src)) { |
|
712 |
movq(dst, as_Address(src)); |
|
713 |
} else { |
|
23491 | 714 |
lea(scratch, src); |
715 |
movq(dst, Address(scratch, 0)); |
|
14626 | 716 |
} |
717 |
} |
|
718 |
} |
|
719 |
||
720 |
void MacroAssembler::movptr(ArrayAddress dst, Register src) { |
|
721 |
movq(as_Address(dst), src); |
|
722 |
} |
|
723 |
||
724 |
void MacroAssembler::movptr(Register dst, ArrayAddress src) { |
|
725 |
movq(dst, as_Address(src)); |
|
726 |
} |
|
727 |
||
728 |
// src should NEVER be a real pointer. Use AddressLiteral for true pointers |
|
729 |
void MacroAssembler::movptr(Address dst, intptr_t src) { |
|
730 |
mov64(rscratch1, src); |
|
731 |
movq(dst, rscratch1); |
|
732 |
} |
|
733 |
||
734 |
// These are mostly for initializing NULL |
|
735 |
void MacroAssembler::movptr(Address dst, int32_t src) { |
|
736 |
movslq(dst, src); |
|
737 |
} |
|
738 |
||
739 |
void MacroAssembler::movptr(Register dst, int32_t src) { |
|
740 |
mov64(dst, (intptr_t)src); |
|
741 |
} |
|
742 |
||
743 |
void MacroAssembler::pushoop(jobject obj) { |
|
744 |
movoop(rscratch1, obj); |
|
745 |
push(rscratch1); |
|
746 |
} |
|
747 |
||
748 |
void MacroAssembler::pushklass(Metadata* obj) { |
|
749 |
mov_metadata(rscratch1, obj); |
|
750 |
push(rscratch1); |
|
751 |
} |
|
752 |
||
753 |
void MacroAssembler::pushptr(AddressLiteral src) { |
|
754 |
lea(rscratch1, src); |
|
755 |
if (src.is_lval()) { |
|
756 |
push(rscratch1); |
|
757 |
} else { |
|
758 |
pushq(Address(rscratch1, 0)); |
|
759 |
} |
|
760 |
} |
|
761 |
||
40644
39e631ed7145
8161598: Kitchensink fails: assert(nm->insts_contains(original_pc)) failed: original PC must be in nmethod/CompiledMethod
dlong
parents:
39256
diff
changeset
|
762 |
void MacroAssembler::reset_last_Java_frame(bool clear_fp) { |
14626 | 763 |
// we must set sp to zero to clear frame |
764 |
movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), NULL_WORD); |
|
765 |
// must clear fp, so that compiled frames are not confused; it is |
|
766 |
// possible that we need it only for debugging |
|
767 |
if (clear_fp) { |
|
768 |
movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), NULL_WORD); |
|
769 |
} |
|
770 |
||
40644
39e631ed7145
8161598: Kitchensink fails: assert(nm->insts_contains(original_pc)) failed: original PC must be in nmethod/CompiledMethod
dlong
parents:
39256
diff
changeset
|
771 |
// Always clear the pc because it could have been set by make_walkable() |
39e631ed7145
8161598: Kitchensink fails: assert(nm->insts_contains(original_pc)) failed: original PC must be in nmethod/CompiledMethod
dlong
parents:
39256
diff
changeset
|
772 |
movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), NULL_WORD); |
46440
61025eecb743
8178811: Minimize the AVX <-> SSE transition penalty through generation of vzeroupper instruction on x86
vdeshpande
parents:
44406
diff
changeset
|
773 |
vzeroupper(); |
14626 | 774 |
} |
775 |
||
776 |
void MacroAssembler::set_last_Java_frame(Register last_java_sp, |
|
777 |
Register last_java_fp, |
|
778 |
address last_java_pc) { |
|
46440
61025eecb743
8178811: Minimize the AVX <-> SSE transition penalty through generation of vzeroupper instruction on x86
vdeshpande
parents:
44406
diff
changeset
|
779 |
vzeroupper(); |
14626 | 780 |
// determine last_java_sp register |
781 |
if (!last_java_sp->is_valid()) { |
|
782 |
last_java_sp = rsp; |
|
783 |
} |
|
784 |
||
785 |
// last_java_fp is optional |
|
786 |
if (last_java_fp->is_valid()) { |
|
787 |
movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), |
|
788 |
last_java_fp); |
|
789 |
} |
|
790 |
||
791 |
// last_java_pc is optional |
|
792 |
if (last_java_pc != NULL) { |
|
793 |
Address java_pc(r15_thread, |
|
794 |
JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset()); |
|
795 |
lea(rscratch1, InternalAddress(last_java_pc)); |
|
796 |
movptr(java_pc, rscratch1); |
|
797 |
} |
|
798 |
||
799 |
movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), last_java_sp); |
|
800 |
} |
|
801 |
||
802 |
static void pass_arg0(MacroAssembler* masm, Register arg) { |
|
803 |
if (c_rarg0 != arg ) { |
|
804 |
masm->mov(c_rarg0, arg); |
|
805 |
} |
|
806 |
} |
|
807 |
||
808 |
static void pass_arg1(MacroAssembler* masm, Register arg) { |
|
809 |
if (c_rarg1 != arg ) { |
|
810 |
masm->mov(c_rarg1, arg); |
|
811 |
} |
|
812 |
} |
|
813 |
||
814 |
static void pass_arg2(MacroAssembler* masm, Register arg) { |
|
815 |
if (c_rarg2 != arg ) { |
|
816 |
masm->mov(c_rarg2, arg); |
|
817 |
} |
|
818 |
} |
|
819 |
||
820 |
static void pass_arg3(MacroAssembler* masm, Register arg) { |
|
821 |
if (c_rarg3 != arg ) { |
|
822 |
masm->mov(c_rarg3, arg); |
|
823 |
} |
|
824 |
} |
|
825 |
||
826 |
void MacroAssembler::stop(const char* msg) { |
|
58536 | 827 |
if (ShowMessageBoxOnError) { |
828 |
address rip = pc(); |
|
829 |
pusha(); // get regs on stack |
|
830 |
lea(c_rarg1, InternalAddress(rip)); |
|
831 |
movq(c_rarg2, rsp); // pass pointer to regs array |
|
832 |
} |
|
14626 | 833 |
lea(c_rarg0, ExternalAddress((address) msg)); |
834 |
andq(rsp, -16); // align stack as required by ABI |
|
835 |
call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64))); |
|
836 |
hlt(); |
|
837 |
} |
|
838 |
||
839 |
void MacroAssembler::warn(const char* msg) { |
|
840 |
push(rbp); |
|
841 |
movq(rbp, rsp); |
|
842 |
andq(rsp, -16); // align stack as required by push_CPU_state and call |
|
843 |
push_CPU_state(); // keeps alignment at 16 bytes |
|
844 |
lea(c_rarg0, ExternalAddress((address) msg)); |
|
48826 | 845 |
lea(rax, ExternalAddress(CAST_FROM_FN_PTR(address, warning))); |
846 |
call(rax); |
|
14626 | 847 |
pop_CPU_state(); |
848 |
mov(rsp, rbp); |
|
849 |
pop(rbp); |
|
850 |
} |
|
851 |
||
852 |
void MacroAssembler::print_state() { |
|
853 |
address rip = pc(); |
|
854 |
pusha(); // get regs on stack |
|
855 |
push(rbp); |
|
856 |
movq(rbp, rsp); |
|
857 |
andq(rsp, -16); // align stack as required by push_CPU_state and call |
|
858 |
push_CPU_state(); // keeps alignment at 16 bytes |
|
859 |
||
860 |
lea(c_rarg0, InternalAddress(rip)); |
|
861 |
lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array |
|
862 |
call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1); |
|
863 |
||
864 |
pop_CPU_state(); |
|
865 |
mov(rsp, rbp); |
|
866 |
pop(rbp); |
|
867 |
popa(); |
|
868 |
} |
|
869 |
||
870 |
#ifndef PRODUCT |
|
871 |
extern "C" void findpc(intptr_t x); |
|
872 |
#endif |
|
873 |
||
874 |
void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) { |
|
875 |
// In order to get locks to work, we need to fake a in_VM state |
|
876 |
if (ShowMessageBoxOnError) { |
|
877 |
JavaThread* thread = JavaThread::current(); |
|
878 |
JavaThreadState saved_state = thread->thread_state(); |
|
879 |
thread->set_thread_state(_thread_in_vm); |
|
880 |
#ifndef PRODUCT |
|
881 |
if (CountBytecodes || TraceBytecodes || StopInterpreterAt) { |
|
882 |
ttyLocker ttyl; |
|
883 |
BytecodeCounter::print(); |
|
884 |
} |
|
885 |
#endif |
|
886 |
// To see where a verify_oop failed, get $ebx+40/X for this frame. |
|
887 |
// XXX correct this offset for amd64 |
|
888 |
// This is the value of eip which points to where verify_oop will return. |
|
889 |
if (os::message_box(msg, "Execution stopped, print registers?")) { |
|
890 |
print_state64(pc, regs); |
|
891 |
BREAKPOINT; |
|
892 |
} |
|
58103
689a80d20550
8230762: Change MacroAssembler::debug32/64 to use fatal instead of assert
chagedorn
parents:
57893
diff
changeset
|
893 |
} |
689a80d20550
8230762: Change MacroAssembler::debug32/64 to use fatal instead of assert
chagedorn
parents:
57893
diff
changeset
|
894 |
fatal("DEBUG MESSAGE: %s", msg); |
14626 | 895 |
} |
896 |
||
897 |
void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) { |
|
898 |
ttyLocker ttyl; |
|
899 |
FlagSetting fs(Debugging, true); |
|
46630
75aa3e39d02c
8182299: Enable disabled clang warnings, build on OSX 10 + Xcode 8
jwilhelm
parents:
46560
diff
changeset
|
900 |
tty->print_cr("rip = 0x%016lx", (intptr_t)pc); |
14626 | 901 |
#ifndef PRODUCT |
902 |
tty->cr(); |
|
903 |
findpc(pc); |
|
904 |
tty->cr(); |
|
905 |
#endif |
|
906 |
#define PRINT_REG(rax, value) \ |
|
907 |
{ tty->print("%s = ", #rax); os::print_location(tty, value); } |
|
908 |
PRINT_REG(rax, regs[15]); |
|
909 |
PRINT_REG(rbx, regs[12]); |
|
910 |
PRINT_REG(rcx, regs[14]); |
|
911 |
PRINT_REG(rdx, regs[13]); |
|
912 |
PRINT_REG(rdi, regs[8]); |
|
913 |
PRINT_REG(rsi, regs[9]); |
|
914 |
PRINT_REG(rbp, regs[10]); |
|
915 |
PRINT_REG(rsp, regs[11]); |
|
916 |
PRINT_REG(r8 , regs[7]); |
|
917 |
PRINT_REG(r9 , regs[6]); |
|
918 |
PRINT_REG(r10, regs[5]); |
|
919 |
PRINT_REG(r11, regs[4]); |
|
920 |
PRINT_REG(r12, regs[3]); |
|
921 |
PRINT_REG(r13, regs[2]); |
|
922 |
PRINT_REG(r14, regs[1]); |
|
923 |
PRINT_REG(r15, regs[0]); |
|
924 |
#undef PRINT_REG |
|
925 |
// Print some words near top of staack. |
|
926 |
int64_t* rsp = (int64_t*) regs[11]; |
|
927 |
int64_t* dump_sp = rsp; |
|
928 |
for (int col1 = 0; col1 < 8; col1++) { |
|
46630
75aa3e39d02c
8182299: Enable disabled clang warnings, build on OSX 10 + Xcode 8
jwilhelm
parents:
46560
diff
changeset
|
929 |
tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); |
14626 | 930 |
os::print_location(tty, *dump_sp++); |
931 |
} |
|
932 |
for (int row = 0; row < 25; row++) { |
|
46630
75aa3e39d02c
8182299: Enable disabled clang warnings, build on OSX 10 + Xcode 8
jwilhelm
parents:
46560
diff
changeset
|
933 |
tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); |
14626 | 934 |
for (int col = 0; col < 4; col++) { |
46630
75aa3e39d02c
8182299: Enable disabled clang warnings, build on OSX 10 + Xcode 8
jwilhelm
parents:
46560
diff
changeset
|
935 |
tty->print(" 0x%016lx", (intptr_t)*dump_sp++); |
14626 | 936 |
} |
937 |
tty->cr(); |
|
938 |
} |
|
939 |
// Print some instructions around pc: |
|
940 |
Disassembler::decode((address)pc-64, (address)pc); |
|
941 |
tty->print_cr("--------"); |
|
942 |
Disassembler::decode((address)pc, (address)pc+32); |
|
943 |
} |
|
944 |
||
945 |
#endif // _LP64 |
|
946 |
||
947 |
// Now versions that are common to 32/64 bit |
|
948 |
||
949 |
void MacroAssembler::addptr(Register dst, int32_t imm32) { |
|
950 |
LP64_ONLY(addq(dst, imm32)) NOT_LP64(addl(dst, imm32)); |
|
951 |
} |
|
952 |
||
953 |
void MacroAssembler::addptr(Register dst, Register src) { |
|
954 |
LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src)); |
|
955 |
} |
|
956 |
||
957 |
void MacroAssembler::addptr(Address dst, Register src) { |
|
958 |
LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src)); |
|
959 |
} |
|
960 |
||
961 |
void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src) { |
|
962 |
if (reachable(src)) { |
|
963 |
Assembler::addsd(dst, as_Address(src)); |
|
964 |
} else { |
|
965 |
lea(rscratch1, src); |
|
966 |
Assembler::addsd(dst, Address(rscratch1, 0)); |
|
967 |
} |
|
968 |
} |
|
969 |
||
970 |
void MacroAssembler::addss(XMMRegister dst, AddressLiteral src) { |
|
971 |
if (reachable(src)) { |
|
972 |
addss(dst, as_Address(src)); |
|
973 |
} else { |
|
974 |
lea(rscratch1, src); |
|
975 |
addss(dst, Address(rscratch1, 0)); |
|
976 |
} |
|
977 |
} |
|
978 |
||
35540
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
979 |
void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src) { |
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
980 |
if (reachable(src)) { |
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
981 |
Assembler::addpd(dst, as_Address(src)); |
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
982 |
} else { |
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
983 |
lea(rscratch1, src); |
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
984 |
Assembler::addpd(dst, Address(rscratch1, 0)); |
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
985 |
} |
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
986 |
} |
e001ad24dcdb
8143353: update for x86 sin and cos in the math lib
vdeshpande
parents:
35146
diff
changeset
|
987 |
|
14626 | 988 |
void MacroAssembler::align(int modulus) { |
32203 | 989 |
align(modulus, offset()); |
990 |
} |
|
991 |
||
992 |
void MacroAssembler::align(int modulus, int target) { |
|
993 |
if (target % modulus != 0) { |
|
994 |
nop(modulus - (target % modulus)); |
|
14626 | 995 |
} |
996 |
} |
|
997 |
||
54750 | 998 |
void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src, Register scratch_reg) { |
14626 | 999 |
// Used in sign-masking with aligned address. |
1000 |
assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); |
|
1001 |
if (reachable(src)) { |
|
1002 |
Assembler::andpd(dst, as_Address(src)); |
|
1003 |
} else { |
|
54750 | 1004 |
lea(scratch_reg, src); |
1005 |
Assembler::andpd(dst, Address(scratch_reg, 0)); |
|
1006 |
} |
|
1007 |
} |
|
1008 |
||
1009 |
void MacroAssembler::andps(XMMRegister dst, AddressLiteral src, Register scratch_reg) { |
|
14626 | 1010 |
// Used in sign-masking with aligned address. |
1011 |
assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); |
|
1012 |
if (reachable(src)) { |
|
1013 |
Assembler::andps(dst, as_Address(src)); |
|
1014 |
} else { |
|
54750 | 1015 |
lea(scratch_reg, src); |
1016 |
Assembler::andps(dst, Address(scratch_reg, 0)); |
|
14626 | 1017 |
} |
1018 |
} |
|
1019 |
||
1020 |
void MacroAssembler::andptr(Register dst, int32_t imm32) { |
|
1021 |
LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32)); |
|
1022 |
} |
|
1023 |
||
23491 | 1024 |
void MacroAssembler::atomic_incl(Address counter_addr) { |
51996
84743156e780
8188764: Obsolete AssumeMP and then remove all support for non-MP builds
dholmes
parents:
51976
diff
changeset
|
1025 |
lock(); |
23491 | 1026 |
incrementl(counter_addr); |
1027 |
} |
|
1028 |
||
1029 |
void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register scr) { |
|
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1030 |
if (reachable(counter_addr)) { |
23491 | 1031 |
atomic_incl(as_Address(counter_addr)); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1032 |
} else { |
23491 | 1033 |
lea(scr, counter_addr); |
1034 |
atomic_incl(Address(scr, 0)); |
|
1035 |
} |
|
1036 |
} |
|
1037 |
||
1038 |
#ifdef _LP64 |
|
1039 |
void MacroAssembler::atomic_incq(Address counter_addr) { |
|
51996
84743156e780
8188764: Obsolete AssumeMP and then remove all support for non-MP builds
dholmes
parents:
51976
diff
changeset
|
1040 |
lock(); |
23491 | 1041 |
incrementq(counter_addr); |
1042 |
} |
|
1043 |
||
1044 |
void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register scr) { |
|
1045 |
if (reachable(counter_addr)) { |
|
1046 |
atomic_incq(as_Address(counter_addr)); |
|
1047 |
} else { |
|
1048 |
lea(scr, counter_addr); |
|
1049 |
atomic_incq(Address(scr, 0)); |
|
1050 |
} |
|
1051 |
} |
|
1052 |
#endif |
|
14626 | 1053 |
|
1054 |
// Writes to stack successive pages until offset reached to check for |
|
1055 |
// stack overflow + shadow pages. This clobbers tmp. |
|
1056 |
void MacroAssembler::bang_stack_size(Register size, Register tmp) { |
|
1057 |
movptr(tmp, rsp); |
|
1058 |
// Bang stack for total size given plus shadow page size. |
|
1059 |
// Bang one page at a time because large size can bang beyond yellow and |
|
1060 |
// red zones. |
|
1061 |
Label loop; |
|
1062 |
bind(loop); |
|
1063 |
movl(Address(tmp, (-os::vm_page_size())), size ); |
|
1064 |
subptr(tmp, os::vm_page_size()); |
|
1065 |
subl(size, os::vm_page_size()); |
|
1066 |
jcc(Assembler::greater, loop); |
|
1067 |
||
1068 |
// Bang down shadow pages too. |
|
21528
479228ecf6ac
8026775: nsk/jvmti/RedefineClasses/StressRedefine crashes due to EXCEPTION_ACCESS_VIOLATION
mikael
parents:
21188
diff
changeset
|
1069 |
// At this point, (tmp-0) is the last address touched, so don't |
479228ecf6ac
8026775: nsk/jvmti/RedefineClasses/StressRedefine crashes due to EXCEPTION_ACCESS_VIOLATION
mikael
parents:
21188
diff
changeset
|
1070 |
// touch it again. (It was touched as (tmp-pagesize) but then tmp |
479228ecf6ac
8026775: nsk/jvmti/RedefineClasses/StressRedefine crashes due to EXCEPTION_ACCESS_VIOLATION
mikael
parents:
21188
diff
changeset
|
1071 |
// was post-decremented.) Skip this address by starting at i=1, and |
479228ecf6ac
8026775: nsk/jvmti/RedefineClasses/StressRedefine crashes due to EXCEPTION_ACCESS_VIOLATION
mikael
parents:
21188
diff
changeset
|
1072 |
// touch a few more pages below. N.B. It is important to touch all |
35201
996db89f378e
8139864: Improve handling of stack protection zones.
goetz
parents:
35071
diff
changeset
|
1073 |
// the way down including all pages in the shadow zone. |
996db89f378e
8139864: Improve handling of stack protection zones.
goetz
parents:
35071
diff
changeset
|
1074 |
for (int i = 1; i < ((int)JavaThread::stack_shadow_zone_size() / os::vm_page_size()); i++) { |
14626 | 1075 |
// this could be any sized move but this is can be a debugging crumb |
1076 |
// so the bigger the better. |
|
1077 |
movptr(Address(tmp, (-i*os::vm_page_size())), size ); |
|
1078 |
} |
|
1079 |
} |
|
1080 |
||
35071
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1081 |
void MacroAssembler::reserved_stack_check() { |
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1082 |
// testing if reserved zone needs to be enabled |
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1083 |
Label no_reserved_zone_enabling; |
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1084 |
Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread); |
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1085 |
NOT_LP64(get_thread(rsi);) |
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1086 |
|
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1087 |
cmpptr(rsp, Address(thread, JavaThread::reserved_stack_activation_offset())); |
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1088 |
jcc(Assembler::below, no_reserved_zone_enabling); |
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1089 |
|
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1090 |
call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), thread); |
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1091 |
jump(RuntimeAddress(StubRoutines::throw_delayed_StackOverflowError_entry())); |
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1092 |
should_not_reach_here(); |
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1093 |
|
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1094 |
bind(no_reserved_zone_enabling); |
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1095 |
} |
a0910b1d3e0d
8046936: JEP 270: Reserved Stack Areas for Critical Sections
fparain
parents:
34633
diff
changeset
|
1096 |
|
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1097 |
int MacroAssembler::biased_locking_enter(Register lock_reg, |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1098 |
Register obj_reg, |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1099 |
Register swap_reg, |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1100 |
Register tmp_reg, |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1101 |
bool swap_reg_contains_mark, |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1102 |
Label& done, |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1103 |
Label* slow_case, |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1104 |
BiasedLockingCounters* counters) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1105 |
assert(UseBiasedLocking, "why call this otherwise?"); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1106 |
assert(swap_reg == rax, "swap_reg must be rax for cmpxchgq"); |
30749
39a2475280ee
6811960: x86 biasedlocking epoch expired rare bug
mockner
parents:
30310
diff
changeset
|
1107 |
assert(tmp_reg != noreg, "tmp_reg must be supplied"); |
39a2475280ee
6811960: x86 biasedlocking epoch expired rare bug
mockner
parents:
30310
diff
changeset
|
1108 |
assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg); |
57777
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1109 |
assert(markWord::age_shift == markWord::lock_bits + markWord::biased_lock_bits, "biased locking makes assumptions about bit layout"); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1110 |
Address mark_addr (obj_reg, oopDesc::mark_offset_in_bytes()); |
37251
9fc139ad74b5
8152358: code and comment cleanups found during the hunt for 8077392
dcubed
parents:
36561
diff
changeset
|
1111 |
NOT_LP64( Address saved_mark_addr(lock_reg, 0); ) |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1112 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1113 |
if (PrintBiasedLockingStatistics && counters == NULL) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1114 |
counters = BiasedLocking::counters(); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1115 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1116 |
// Biased locking |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1117 |
// See whether the lock is currently biased toward our thread and |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1118 |
// whether the epoch is still valid |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1119 |
// Note that the runtime guarantees sufficient alignment of JavaThread |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1120 |
// pointers to allow age to be placed into low bits |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1121 |
// First check to see whether biasing is even enabled for this object |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1122 |
Label cas_label; |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1123 |
int null_check_offset = -1; |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1124 |
if (!swap_reg_contains_mark) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1125 |
null_check_offset = offset(); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1126 |
movptr(swap_reg, mark_addr); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1127 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1128 |
movptr(tmp_reg, swap_reg); |
57777
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1129 |
andptr(tmp_reg, markWord::biased_lock_mask_in_place); |
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1130 |
cmpptr(tmp_reg, markWord::biased_lock_pattern); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1131 |
jcc(Assembler::notEqual, cas_label); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1132 |
// The bias pattern is present in the object's header. Need to check |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1133 |
// whether the bias owner and the epoch are both still current. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1134 |
#ifndef _LP64 |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1135 |
// Note that because there is no current thread register on x86_32 we |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1136 |
// need to store off the mark word we read out of the object to |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1137 |
// avoid reloading it and needing to recheck invariants below. This |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1138 |
// store is unfortunate but it makes the overall code shorter and |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1139 |
// simpler. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1140 |
movptr(saved_mark_addr, swap_reg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1141 |
#endif |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1142 |
if (swap_reg_contains_mark) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1143 |
null_check_offset = offset(); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1144 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1145 |
load_prototype_header(tmp_reg, obj_reg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1146 |
#ifdef _LP64 |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1147 |
orptr(tmp_reg, r15_thread); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1148 |
xorptr(tmp_reg, swap_reg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1149 |
Register header_reg = tmp_reg; |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1150 |
#else |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1151 |
xorptr(tmp_reg, swap_reg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1152 |
get_thread(swap_reg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1153 |
xorptr(swap_reg, tmp_reg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1154 |
Register header_reg = swap_reg; |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1155 |
#endif |
57777
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1156 |
andptr(header_reg, ~((int) markWord::age_mask_in_place)); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1157 |
if (counters != NULL) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1158 |
cond_inc32(Assembler::zero, |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1159 |
ExternalAddress((address) counters->biased_lock_entry_count_addr())); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1160 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1161 |
jcc(Assembler::equal, done); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1162 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1163 |
Label try_revoke_bias; |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1164 |
Label try_rebias; |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1165 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1166 |
// At this point we know that the header has the bias pattern and |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1167 |
// that we are not the bias owner in the current epoch. We need to |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1168 |
// figure out more details about the state of the header in order to |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1169 |
// know what operations can be legally performed on the object's |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1170 |
// header. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1171 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1172 |
// If the low three bits in the xor result aren't clear, that means |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1173 |
// the prototype header is no longer biased and we have to revoke |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1174 |
// the bias on this object. |
57777
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1175 |
testptr(header_reg, markWord::biased_lock_mask_in_place); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1176 |
jccb(Assembler::notZero, try_revoke_bias); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1177 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1178 |
// Biasing is still enabled for this data type. See whether the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1179 |
// epoch of the current bias is still valid, meaning that the epoch |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1180 |
// bits of the mark word are equal to the epoch bits of the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1181 |
// prototype header. (Note that the prototype header's epoch bits |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1182 |
// only change at a safepoint.) If not, attempt to rebias the object |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1183 |
// toward the current thread. Note that we must be absolutely sure |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1184 |
// that the current epoch is invalid in order to do this because |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1185 |
// otherwise the manipulations it performs on the mark word are |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1186 |
// illegal. |
57777
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1187 |
testptr(header_reg, markWord::epoch_mask_in_place); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1188 |
jccb(Assembler::notZero, try_rebias); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1189 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1190 |
// The epoch of the current bias is still valid but we know nothing |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1191 |
// about the owner; it might be set or it might be clear. Try to |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1192 |
// acquire the bias of the object using an atomic operation. If this |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1193 |
// fails we will go in to the runtime to revoke the object's bias. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1194 |
// Note that we first construct the presumed unbiased header so we |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1195 |
// don't accidentally blow away another thread's valid bias. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1196 |
NOT_LP64( movptr(swap_reg, saved_mark_addr); ) |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1197 |
andptr(swap_reg, |
57777
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1198 |
markWord::biased_lock_mask_in_place | markWord::age_mask_in_place | markWord::epoch_mask_in_place); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1199 |
#ifdef _LP64 |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1200 |
movptr(tmp_reg, swap_reg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1201 |
orptr(tmp_reg, r15_thread); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1202 |
#else |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1203 |
get_thread(tmp_reg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1204 |
orptr(tmp_reg, swap_reg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1205 |
#endif |
51996
84743156e780
8188764: Obsolete AssumeMP and then remove all support for non-MP builds
dholmes
parents:
51976
diff
changeset
|
1206 |
lock(); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1207 |
cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1208 |
// If the biasing toward our thread failed, this means that |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1209 |
// another thread succeeded in biasing it toward itself and we |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1210 |
// need to revoke that bias. The revocation will occur in the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1211 |
// interpreter runtime in the slow case. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1212 |
if (counters != NULL) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1213 |
cond_inc32(Assembler::zero, |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1214 |
ExternalAddress((address) counters->anonymously_biased_lock_entry_count_addr())); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1215 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1216 |
if (slow_case != NULL) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1217 |
jcc(Assembler::notZero, *slow_case); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1218 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1219 |
jmp(done); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1220 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1221 |
bind(try_rebias); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1222 |
// At this point we know the epoch has expired, meaning that the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1223 |
// current "bias owner", if any, is actually invalid. Under these |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1224 |
// circumstances _only_, we are allowed to use the current header's |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1225 |
// value as the comparison value when doing the cas to acquire the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1226 |
// bias in the current epoch. In other words, we allow transfer of |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1227 |
// the bias from one thread to another directly in this situation. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1228 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1229 |
// FIXME: due to a lack of registers we currently blow away the age |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1230 |
// bits in this situation. Should attempt to preserve them. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1231 |
load_prototype_header(tmp_reg, obj_reg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1232 |
#ifdef _LP64 |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1233 |
orptr(tmp_reg, r15_thread); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1234 |
#else |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1235 |
get_thread(swap_reg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1236 |
orptr(tmp_reg, swap_reg); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1237 |
movptr(swap_reg, saved_mark_addr); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1238 |
#endif |
51996
84743156e780
8188764: Obsolete AssumeMP and then remove all support for non-MP builds
dholmes
parents:
51976
diff
changeset
|
1239 |
lock(); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1240 |
cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1241 |
// If the biasing toward our thread failed, then another thread |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1242 |
// succeeded in biasing it toward itself and we need to revoke that |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1243 |
// bias. The revocation will occur in the runtime in the slow case. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1244 |
if (counters != NULL) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1245 |
cond_inc32(Assembler::zero, |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1246 |
ExternalAddress((address) counters->rebiased_lock_entry_count_addr())); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1247 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1248 |
if (slow_case != NULL) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1249 |
jcc(Assembler::notZero, *slow_case); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1250 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1251 |
jmp(done); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1252 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1253 |
bind(try_revoke_bias); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1254 |
// The prototype mark in the klass doesn't have the bias bit set any |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1255 |
// more, indicating that objects of this data type are not supposed |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1256 |
// to be biased any more. We are going to try to reset the mark of |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1257 |
// this object to the prototype value and fall through to the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1258 |
// CAS-based locking scheme. Note that if our CAS fails, it means |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1259 |
// that another thread raced us for the privilege of revoking the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1260 |
// bias of this particular object, so it's okay to continue in the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1261 |
// normal locking code. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1262 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1263 |
// FIXME: due to a lack of registers we currently blow away the age |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1264 |
// bits in this situation. Should attempt to preserve them. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1265 |
NOT_LP64( movptr(swap_reg, saved_mark_addr); ) |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1266 |
load_prototype_header(tmp_reg, obj_reg); |
51996
84743156e780
8188764: Obsolete AssumeMP and then remove all support for non-MP builds
dholmes
parents:
51976
diff
changeset
|
1267 |
lock(); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1268 |
cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1269 |
// Fall through to the normal CAS-based lock, because no matter what |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1270 |
// the result of the above CAS, some thread must have succeeded in |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1271 |
// removing the bias bit from the object's header. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1272 |
if (counters != NULL) { |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1273 |
cond_inc32(Assembler::zero, |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1274 |
ExternalAddress((address) counters->revoked_lock_entry_count_addr())); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1275 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1276 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1277 |
bind(cas_label); |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1278 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1279 |
return null_check_offset; |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1280 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1281 |
|
14626 | 1282 |
void MacroAssembler::biased_locking_exit(Register obj_reg, Register temp_reg, Label& done) { |
1283 |
assert(UseBiasedLocking, "why call this otherwise?"); |
|
1284 |
||
1285 |
// Check for biased locking unlock case, which is a no-op |
|
1286 |
// Note: we do not have to check the thread ID for two reasons. |
|
1287 |
// First, the interpreter checks for IllegalMonitorStateException at |
|
1288 |
// a higher level. Second, if the bias was revoked while we held the |
|
1289 |
// lock, the object could not be rebiased toward another thread, so |
|
1290 |
// the bias bit would be clear. |
|
1291 |
movptr(temp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes())); |
|
57777
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1292 |
andptr(temp_reg, markWord::biased_lock_mask_in_place); |
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1293 |
cmpptr(temp_reg, markWord::biased_lock_pattern); |
14626 | 1294 |
jcc(Assembler::equal, done); |
1295 |
} |
|
1296 |
||
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1297 |
#ifdef COMPILER2 |
23491 | 1298 |
|
1299 |
#if INCLUDE_RTM_OPT |
|
1300 |
||
1301 |
// Update rtm_counters based on abort status |
|
1302 |
// input: abort_status |
|
1303 |
// rtm_counters (RTMLockingCounters*) |
|
1304 |
// flags are killed |
|
1305 |
void MacroAssembler::rtm_counters_update(Register abort_status, Register rtm_counters) { |
|
1306 |
||
1307 |
atomic_incptr(Address(rtm_counters, RTMLockingCounters::abort_count_offset())); |
|
1308 |
if (PrintPreciseRTMLockingStatistics) { |
|
1309 |
for (int i = 0; i < RTMLockingCounters::ABORT_STATUS_LIMIT; i++) { |
|
1310 |
Label check_abort; |
|
1311 |
testl(abort_status, (1<<i)); |
|
1312 |
jccb(Assembler::equal, check_abort); |
|
1313 |
atomic_incptr(Address(rtm_counters, RTMLockingCounters::abortX_count_offset() + (i * sizeof(uintx)))); |
|
1314 |
bind(check_abort); |
|
1315 |
} |
|
1316 |
} |
|
1317 |
} |
|
1318 |
||
1319 |
// Branch if (random & (count-1) != 0), count is 2^n |
|
1320 |
// tmp, scr and flags are killed |
|
1321 |
void MacroAssembler::branch_on_random_using_rdtsc(Register tmp, Register scr, int count, Label& brLabel) { |
|
1322 |
assert(tmp == rax, ""); |
|
1323 |
assert(scr == rdx, ""); |
|
1324 |
rdtsc(); // modifies EDX:EAX |
|
1325 |
andptr(tmp, count-1); |
|
1326 |
jccb(Assembler::notZero, brLabel); |
|
1327 |
} |
|
1328 |
||
1329 |
// Perform abort ratio calculation, set no_rtm bit if high ratio |
|
1330 |
// input: rtm_counters_Reg (RTMLockingCounters* address) |
|
1331 |
// tmpReg, rtm_counters_Reg and flags are killed |
|
1332 |
void MacroAssembler::rtm_abort_ratio_calculation(Register tmpReg, |
|
1333 |
Register rtm_counters_Reg, |
|
1334 |
RTMLockingCounters* rtm_counters, |
|
1335 |
Metadata* method_data) { |
|
1336 |
Label L_done, L_check_always_rtm1, L_check_always_rtm2; |
|
1337 |
||
1338 |
if (RTMLockingCalculationDelay > 0) { |
|
1339 |
// Delay calculation |
|
1340 |
movptr(tmpReg, ExternalAddress((address) RTMLockingCounters::rtm_calculation_flag_addr()), tmpReg); |
|
1341 |
testptr(tmpReg, tmpReg); |
|
1342 |
jccb(Assembler::equal, L_done); |
|
1343 |
} |
|
1344 |
// Abort ratio calculation only if abort_count > RTMAbortThreshold |
|
1345 |
// Aborted transactions = abort_count * 100 |
|
1346 |
// All transactions = total_count * RTMTotalCountIncrRate |
|
1347 |
// Set no_rtm bit if (Aborted transactions >= All transactions * RTMAbortRatio) |
|
1348 |
||
1349 |
movptr(tmpReg, Address(rtm_counters_Reg, RTMLockingCounters::abort_count_offset())); |
|
1350 |
cmpptr(tmpReg, RTMAbortThreshold); |
|
1351 |
jccb(Assembler::below, L_check_always_rtm2); |
|
1352 |
imulptr(tmpReg, tmpReg, 100); |
|
1353 |
||
1354 |
Register scrReg = rtm_counters_Reg; |
|
1355 |
movptr(scrReg, Address(rtm_counters_Reg, RTMLockingCounters::total_count_offset())); |
|
1356 |
imulptr(scrReg, scrReg, RTMTotalCountIncrRate); |
|
1357 |
imulptr(scrReg, scrReg, RTMAbortRatio); |
|
1358 |
cmpptr(tmpReg, scrReg); |
|
1359 |
jccb(Assembler::below, L_check_always_rtm1); |
|
1360 |
if (method_data != NULL) { |
|
1361 |
// set rtm_state to "no rtm" in MDO |
|
1362 |
mov_metadata(tmpReg, method_data); |
|
51996
84743156e780
8188764: Obsolete AssumeMP and then remove all support for non-MP builds
dholmes
parents:
51976
diff
changeset
|
1363 |
lock(); |
23491 | 1364 |
orl(Address(tmpReg, MethodData::rtm_state_offset_in_bytes()), NoRTM); |
1365 |
} |
|
1366 |
jmpb(L_done); |
|
1367 |
bind(L_check_always_rtm1); |
|
1368 |
// Reload RTMLockingCounters* address |
|
1369 |
lea(rtm_counters_Reg, ExternalAddress((address)rtm_counters)); |
|
1370 |
bind(L_check_always_rtm2); |
|
1371 |
movptr(tmpReg, Address(rtm_counters_Reg, RTMLockingCounters::total_count_offset())); |
|
1372 |
cmpptr(tmpReg, RTMLockingThreshold / RTMTotalCountIncrRate); |
|
1373 |
jccb(Assembler::below, L_done); |
|
1374 |
if (method_data != NULL) { |
|
1375 |
// set rtm_state to "always rtm" in MDO |
|
1376 |
mov_metadata(tmpReg, method_data); |
|
51996
84743156e780
8188764: Obsolete AssumeMP and then remove all support for non-MP builds
dholmes
parents:
51976
diff
changeset
|
1377 |
lock(); |
23491 | 1378 |
orl(Address(tmpReg, MethodData::rtm_state_offset_in_bytes()), UseRTM); |
1379 |
} |
|
1380 |
bind(L_done); |
|
1381 |
} |
|
1382 |
||
1383 |
// Update counters and perform abort ratio calculation |
|
1384 |
// input: abort_status_Reg |
|
1385 |
// rtm_counters_Reg, flags are killed |
|
1386 |
void MacroAssembler::rtm_profiling(Register abort_status_Reg, |
|
1387 |
Register rtm_counters_Reg, |
|
1388 |
RTMLockingCounters* rtm_counters, |
|
1389 |
Metadata* method_data, |
|
1390 |
bool profile_rtm) { |
|
1391 |
||
1392 |
assert(rtm_counters != NULL, "should not be NULL when profiling RTM"); |
|
1393 |
// update rtm counters based on rax value at abort |
|
1394 |
// reads abort_status_Reg, updates flags |
|
1395 |
lea(rtm_counters_Reg, ExternalAddress((address)rtm_counters)); |
|
1396 |
rtm_counters_update(abort_status_Reg, rtm_counters_Reg); |
|
1397 |
if (profile_rtm) { |
|
1398 |
// Save abort status because abort_status_Reg is used by following code. |
|
1399 |
if (RTMRetryCount > 0) { |
|
1400 |
push(abort_status_Reg); |
|
1401 |
} |
|
1402 |
assert(rtm_counters != NULL, "should not be NULL when profiling RTM"); |
|
1403 |
rtm_abort_ratio_calculation(abort_status_Reg, rtm_counters_Reg, rtm_counters, method_data); |
|
1404 |
// restore abort status |
|
1405 |
if (RTMRetryCount > 0) { |
|
1406 |
pop(abort_status_Reg); |
|
1407 |
} |
|
1408 |
} |
|
1409 |
} |
|
1410 |
||
1411 |
// Retry on abort if abort's status is 0x6: can retry (0x2) | memory conflict (0x4) |
|
1412 |
// inputs: retry_count_Reg |
|
1413 |
// : abort_status_Reg |
|
1414 |
// output: retry_count_Reg decremented by 1 |
|
1415 |
// flags are killed |
|
1416 |
void MacroAssembler::rtm_retry_lock_on_abort(Register retry_count_Reg, Register abort_status_Reg, Label& retryLabel) { |
|
1417 |
Label doneRetry; |
|
1418 |
assert(abort_status_Reg == rax, ""); |
|
1419 |
// The abort reason bits are in eax (see all states in rtmLocking.hpp) |
|
1420 |
// 0x6 = conflict on which we can retry (0x2) | memory conflict (0x4) |
|
1421 |
// if reason is in 0x6 and retry count != 0 then retry |
|
1422 |
andptr(abort_status_Reg, 0x6); |
|
1423 |
jccb(Assembler::zero, doneRetry); |
|
1424 |
testl(retry_count_Reg, retry_count_Reg); |
|
1425 |
jccb(Assembler::zero, doneRetry); |
|
1426 |
pause(); |
|
1427 |
decrementl(retry_count_Reg); |
|
1428 |
jmp(retryLabel); |
|
1429 |
bind(doneRetry); |
|
1430 |
} |
|
1431 |
||
1432 |
// Spin and retry if lock is busy, |
|
1433 |
// inputs: box_Reg (monitor address) |
|
1434 |
// : retry_count_Reg |
|
1435 |
// output: retry_count_Reg decremented by 1 |
|
1436 |
// : clear z flag if retry count exceeded |
|
1437 |
// tmp_Reg, scr_Reg, flags are killed |
|
1438 |
void MacroAssembler::rtm_retry_lock_on_busy(Register retry_count_Reg, Register box_Reg, |
|
1439 |
Register tmp_Reg, Register scr_Reg, Label& retryLabel) { |
|
1440 |
Label SpinLoop, SpinExit, doneRetry; |
|
27608 | 1441 |
int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner); |
23491 | 1442 |
|
1443 |
testl(retry_count_Reg, retry_count_Reg); |
|
1444 |
jccb(Assembler::zero, doneRetry); |
|
1445 |
decrementl(retry_count_Reg); |
|
1446 |
movptr(scr_Reg, RTMSpinLoopCount); |
|
1447 |
||
1448 |
bind(SpinLoop); |
|
1449 |
pause(); |
|
1450 |
decrementl(scr_Reg); |
|
1451 |
jccb(Assembler::lessEqual, SpinExit); |
|
1452 |
movptr(tmp_Reg, Address(box_Reg, owner_offset)); |
|
1453 |
testptr(tmp_Reg, tmp_Reg); |
|
1454 |
jccb(Assembler::notZero, SpinLoop); |
|
1455 |
||
1456 |
bind(SpinExit); |
|
1457 |
jmp(retryLabel); |
|
1458 |
bind(doneRetry); |
|
1459 |
incrementl(retry_count_Reg); // clear z flag |
|
1460 |
} |
|
1461 |
||
1462 |
// Use RTM for normal stack locks |
|
1463 |
// Input: objReg (object to lock) |
|
1464 |
void MacroAssembler::rtm_stack_locking(Register objReg, Register tmpReg, Register scrReg, |
|
1465 |
Register retry_on_abort_count_Reg, |
|
1466 |
RTMLockingCounters* stack_rtm_counters, |
|
1467 |
Metadata* method_data, bool profile_rtm, |
|
1468 |
Label& DONE_LABEL, Label& IsInflated) { |
|
1469 |
assert(UseRTMForStackLocks, "why call this otherwise?"); |
|
1470 |
assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking"); |
|
1471 |
assert(tmpReg == rax, ""); |
|
1472 |
assert(scrReg == rdx, ""); |
|
1473 |
Label L_rtm_retry, L_decrement_retry, L_on_abort; |
|
1474 |
||
1475 |
if (RTMRetryCount > 0) { |
|
1476 |
movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort |
|
1477 |
bind(L_rtm_retry); |
|
1478 |
} |
|
46449
7b2416f0f524
8167659: Access of mark word should use oopDesc::mark_offset_in_bytes() instead of '0'
rkennke
parents:
46440
diff
changeset
|
1479 |
movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); |
57777
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1480 |
testptr(tmpReg, markWord::monitor_value); // inflated vs stack-locked|neutral|biased |
23847
d792e42aeb4f
8038939: Some options related to RTM locking optimization works inconsistently
kvn
parents:
23491
diff
changeset
|
1481 |
jcc(Assembler::notZero, IsInflated); |
d792e42aeb4f
8038939: Some options related to RTM locking optimization works inconsistently
kvn
parents:
23491
diff
changeset
|
1482 |
|
23491 | 1483 |
if (PrintPreciseRTMLockingStatistics || profile_rtm) { |
1484 |
Label L_noincrement; |
|
1485 |
if (RTMTotalCountIncrRate > 1) { |
|
1486 |
// tmpReg, scrReg and flags are killed |
|
46494
3fdd343bc5ea
8180612: [ppc] assert failure in cpu/ppc/vm/assembler_ppc.hpp due to immediate value out of range
lucy
parents:
46449
diff
changeset
|
1487 |
branch_on_random_using_rdtsc(tmpReg, scrReg, RTMTotalCountIncrRate, L_noincrement); |
23491 | 1488 |
} |
1489 |
assert(stack_rtm_counters != NULL, "should not be NULL when profiling RTM"); |
|
1490 |
atomic_incptr(ExternalAddress((address)stack_rtm_counters->total_count_addr()), scrReg); |
|
1491 |
bind(L_noincrement); |
|
1492 |
} |
|
1493 |
xbegin(L_on_abort); |
|
46449
7b2416f0f524
8167659: Access of mark word should use oopDesc::mark_offset_in_bytes() instead of '0'
rkennke
parents:
46440
diff
changeset
|
1494 |
movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // fetch markword |
57777
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1495 |
andptr(tmpReg, markWord::biased_lock_mask_in_place); // look at 3 lock bits |
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1496 |
cmpptr(tmpReg, markWord::unlocked_value); // bits = 001 unlocked |
23491 | 1497 |
jcc(Assembler::equal, DONE_LABEL); // all done if unlocked |
1498 |
||
1499 |
Register abort_status_Reg = tmpReg; // status of abort is stored in RAX |
|
1500 |
if (UseRTMXendForLockBusy) { |
|
1501 |
xend(); |
|
23847
d792e42aeb4f
8038939: Some options related to RTM locking optimization works inconsistently
kvn
parents:
23491
diff
changeset
|
1502 |
movptr(abort_status_Reg, 0x2); // Set the abort status to 2 (so we can retry) |
23491 | 1503 |
jmp(L_decrement_retry); |
1504 |
} |
|
1505 |
else { |
|
1506 |
xabort(0); |
|
1507 |
} |
|
1508 |
bind(L_on_abort); |
|
1509 |
if (PrintPreciseRTMLockingStatistics || profile_rtm) { |
|
1510 |
rtm_profiling(abort_status_Reg, scrReg, stack_rtm_counters, method_data, profile_rtm); |
|
1511 |
} |
|
1512 |
bind(L_decrement_retry); |
|
1513 |
if (RTMRetryCount > 0) { |
|
1514 |
// retry on lock abort if abort status is 'can retry' (0x2) or 'memory conflict' (0x4) |
|
1515 |
rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry); |
|
1516 |
} |
|
1517 |
} |
|
1518 |
||
1519 |
// Use RTM for inflating locks |
|
1520 |
// inputs: objReg (object to lock) |
|
1521 |
// boxReg (on-stack box address (displaced header location) - KILLED) |
|
57777
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1522 |
// tmpReg (ObjectMonitor address + markWord::monitor_value) |
23491 | 1523 |
void MacroAssembler::rtm_inflated_locking(Register objReg, Register boxReg, Register tmpReg, |
1524 |
Register scrReg, Register retry_on_busy_count_Reg, |
|
1525 |
Register retry_on_abort_count_Reg, |
|
1526 |
RTMLockingCounters* rtm_counters, |
|
1527 |
Metadata* method_data, bool profile_rtm, |
|
1528 |
Label& DONE_LABEL) { |
|
1529 |
assert(UseRTMLocking, "why call this otherwise?"); |
|
1530 |
assert(tmpReg == rax, ""); |
|
1531 |
assert(scrReg == rdx, ""); |
|
1532 |
Label L_rtm_retry, L_decrement_retry, L_on_abort; |
|
27608 | 1533 |
int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner); |
23491 | 1534 |
|
1535 |
// Without cast to int32_t a movptr will destroy r10 which is typically obj |
|
57777
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1536 |
movptr(Address(boxReg, 0), (int32_t)intptr_t(markWord::unused_mark().value())); |
23491 | 1537 |
movptr(boxReg, tmpReg); // Save ObjectMonitor address |
1538 |
||
1539 |
if (RTMRetryCount > 0) { |
|
1540 |
movl(retry_on_busy_count_Reg, RTMRetryCount); // Retry on lock busy |
|
1541 |
movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort |
|
1542 |
bind(L_rtm_retry); |
|
1543 |
} |
|
1544 |
if (PrintPreciseRTMLockingStatistics || profile_rtm) { |
|
1545 |
Label L_noincrement; |
|
1546 |
if (RTMTotalCountIncrRate > 1) { |
|
1547 |
// tmpReg, scrReg and flags are killed |
|
46494
3fdd343bc5ea
8180612: [ppc] assert failure in cpu/ppc/vm/assembler_ppc.hpp due to immediate value out of range
lucy
parents:
46449
diff
changeset
|
1548 |
branch_on_random_using_rdtsc(tmpReg, scrReg, RTMTotalCountIncrRate, L_noincrement); |
23491 | 1549 |
} |
1550 |
assert(rtm_counters != NULL, "should not be NULL when profiling RTM"); |
|
1551 |
atomic_incptr(ExternalAddress((address)rtm_counters->total_count_addr()), scrReg); |
|
1552 |
bind(L_noincrement); |
|
1553 |
} |
|
1554 |
xbegin(L_on_abort); |
|
46449
7b2416f0f524
8167659: Access of mark word should use oopDesc::mark_offset_in_bytes() instead of '0'
rkennke
parents:
46440
diff
changeset
|
1555 |
movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); |
23491 | 1556 |
movptr(tmpReg, Address(tmpReg, owner_offset)); |
1557 |
testptr(tmpReg, tmpReg); |
|
1558 |
jcc(Assembler::zero, DONE_LABEL); |
|
1559 |
if (UseRTMXendForLockBusy) { |
|
1560 |
xend(); |
|
1561 |
jmp(L_decrement_retry); |
|
1562 |
} |
|
1563 |
else { |
|
1564 |
xabort(0); |
|
1565 |
} |
|
1566 |
bind(L_on_abort); |
|
1567 |
Register abort_status_Reg = tmpReg; // status of abort is stored in RAX |
|
1568 |
if (PrintPreciseRTMLockingStatistics || profile_rtm) { |
|
1569 |
rtm_profiling(abort_status_Reg, scrReg, rtm_counters, method_data, profile_rtm); |
|
1570 |
} |
|
1571 |
if (RTMRetryCount > 0) { |
|
1572 |
// retry on lock abort if abort status is 'can retry' (0x2) or 'memory conflict' (0x4) |
|
1573 |
rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry); |
|
1574 |
} |
|
1575 |
||
1576 |
movptr(tmpReg, Address(boxReg, owner_offset)) ; |
|
1577 |
testptr(tmpReg, tmpReg) ; |
|
1578 |
jccb(Assembler::notZero, L_decrement_retry) ; |
|
1579 |
||
1580 |
// Appears unlocked - try to swing _owner from null to non-null. |
|
1581 |
// Invariant: tmpReg == 0. tmpReg is EAX which is the implicit cmpxchg comparand. |
|
1582 |
#ifdef _LP64 |
|
1583 |
Register threadReg = r15_thread; |
|
1584 |
#else |
|
1585 |
get_thread(scrReg); |
|
1586 |
Register threadReg = scrReg; |
|
1587 |
#endif |
|
51996
84743156e780
8188764: Obsolete AssumeMP and then remove all support for non-MP builds
dholmes
parents:
51976
diff
changeset
|
1588 |
lock(); |
23491 | 1589 |
cmpxchgptr(threadReg, Address(boxReg, owner_offset)); // Updates tmpReg |
1590 |
||
1591 |
if (RTMRetryCount > 0) { |
|
1592 |
// success done else retry |
|
1593 |
jccb(Assembler::equal, DONE_LABEL) ; |
|
1594 |
bind(L_decrement_retry); |
|
1595 |
// Spin and retry if lock is busy. |
|
1596 |
rtm_retry_lock_on_busy(retry_on_busy_count_Reg, boxReg, tmpReg, scrReg, L_rtm_retry); |
|
1597 |
} |
|
1598 |
else { |
|
1599 |
bind(L_decrement_retry); |
|
1600 |
} |
|
1601 |
} |
|
1602 |
||
1603 |
#endif // INCLUDE_RTM_OPT |
|
1604 |
||
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1605 |
// Fast_Lock and Fast_Unlock used by C2 |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1606 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1607 |
// Because the transitions from emitted code to the runtime |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1608 |
// monitorenter/exit helper stubs are so slow it's critical that |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1609 |
// we inline both the stack-locking fast-path and the inflated fast path. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1610 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1611 |
// See also: cmpFastLock and cmpFastUnlock. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1612 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1613 |
// What follows is a specialized inline transliteration of the code |
57893
49fea19f0726
8229844: Remove attempt_rebias parameter from revoke_and_rebias()
pchilanomate
parents:
57804
diff
changeset
|
1614 |
// in enter() and exit(). If we're concerned about I$ bloat another |
49fea19f0726
8229844: Remove attempt_rebias parameter from revoke_and_rebias()
pchilanomate
parents:
57804
diff
changeset
|
1615 |
// option would be to emit TrySlowEnter and TrySlowExit methods |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1616 |
// at startup-time. These methods would accept arguments as |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1617 |
// (rax,=Obj, rbx=Self, rcx=box, rdx=Scratch) and return success-failure |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1618 |
// indications in the icc.ZFlag. Fast_Lock and Fast_Unlock would simply |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1619 |
// marshal the arguments and emit calls to TrySlowEnter and TrySlowExit. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1620 |
// In practice, however, the # of lock sites is bounded and is usually small. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1621 |
// Besides the call overhead, TrySlowEnter and TrySlowExit might suffer |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1622 |
// if the processor uses simple bimodal branch predictors keyed by EIP |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1623 |
// Since the helper routines would be called from multiple synchronization |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1624 |
// sites. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1625 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1626 |
// An even better approach would be write "MonitorEnter()" and "MonitorExit()" |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1627 |
// in java - using j.u.c and unsafe - and just bind the lock and unlock sites |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1628 |
// to those specialized methods. That'd give us a mostly platform-independent |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1629 |
// implementation that the JITs could optimize and inline at their pleasure. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1630 |
// Done correctly, the only time we'd need to cross to native could would be |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1631 |
// to park() or unpark() threads. We'd also need a few more unsafe operators |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1632 |
// to (a) prevent compiler-JIT reordering of non-volatile accesses, and |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1633 |
// (b) explicit barriers or fence operations. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1634 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1635 |
// TODO: |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1636 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1637 |
// * Arrange for C2 to pass "Self" into Fast_Lock and Fast_Unlock in one of the registers (scr). |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1638 |
// This avoids manifesting the Self pointer in the Fast_Lock and Fast_Unlock terminals. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1639 |
// Given TLAB allocation, Self is usually manifested in a register, so passing it into |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1640 |
// the lock operators would typically be faster than reifying Self. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1641 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1642 |
// * Ideally I'd define the primitives as: |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1643 |
// fast_lock (nax Obj, nax box, EAX tmp, nax scr) where box, tmp and scr are KILLED. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1644 |
// fast_unlock (nax Obj, EAX box, nax tmp) where box and tmp are KILLED |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1645 |
// Unfortunately ADLC bugs prevent us from expressing the ideal form. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1646 |
// Instead, we're stuck with a rather awkward and brittle register assignments below. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1647 |
// Furthermore the register assignments are overconstrained, possibly resulting in |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1648 |
// sub-optimal code near the synchronization site. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1649 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1650 |
// * Eliminate the sp-proximity tests and just use "== Self" tests instead. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1651 |
// Alternately, use a better sp-proximity test. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1652 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1653 |
// * Currently ObjectMonitor._Owner can hold either an sp value or a (THREAD *) value. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1654 |
// Either one is sufficient to uniquely identify a thread. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1655 |
// TODO: eliminate use of sp in _owner and use get_thread(tr) instead. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1656 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1657 |
// * Intrinsify notify() and notifyAll() for the common cases where the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1658 |
// object is locked by the calling thread but the waitlist is empty. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1659 |
// avoid the expensive JNI call to JVM_Notify() and JVM_NotifyAll(). |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1660 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1661 |
// * use jccb and jmpb instead of jcc and jmp to improve code density. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1662 |
// But beware of excessive branch density on AMD Opterons. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1663 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1664 |
// * Both Fast_Lock and Fast_Unlock set the ICC.ZF to indicate success |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1665 |
// or failure of the fast-path. If the fast-path fails then we pass |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1666 |
// control to the slow-path, typically in C. In Fast_Lock and |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1667 |
// Fast_Unlock we often branch to DONE_LABEL, just to find that C2 |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1668 |
// will emit a conditional branch immediately after the node. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1669 |
// So we have branches to branches and lots of ICC.ZF games. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1670 |
// Instead, it might be better to have C2 pass a "FailureLabel" |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1671 |
// into Fast_Lock and Fast_Unlock. In the case of success, control |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1672 |
// will drop through the node. ICC.ZF is undefined at exit. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1673 |
// In the case of failure, the node will branch directly to the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1674 |
// FailureLabel |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1675 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1676 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1677 |
// obj: object to lock |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1678 |
// box: on-stack box address (displaced header location) - KILLED |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1679 |
// rax,: tmp -- KILLED |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1680 |
// scr: tmp -- KILLED |
23491 | 1681 |
void MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmpReg, |
1682 |
Register scrReg, Register cx1Reg, Register cx2Reg, |
|
1683 |
BiasedLockingCounters* counters, |
|
1684 |
RTMLockingCounters* rtm_counters, |
|
1685 |
RTMLockingCounters* stack_rtm_counters, |
|
1686 |
Metadata* method_data, |
|
1687 |
bool use_rtm, bool profile_rtm) { |
|
37251
9fc139ad74b5
8152358: code and comment cleanups found during the hunt for 8077392
dcubed
parents:
36561
diff
changeset
|
1688 |
// Ensure the register assignments are disjoint |
23491 | 1689 |
assert(tmpReg == rax, ""); |
1690 |
||
1691 |
if (use_rtm) { |
|
1692 |
assert_different_registers(objReg, boxReg, tmpReg, scrReg, cx1Reg, cx2Reg); |
|
1693 |
} else { |
|
1694 |
assert(cx1Reg == noreg, ""); |
|
1695 |
assert(cx2Reg == noreg, ""); |
|
1696 |
assert_different_registers(objReg, boxReg, tmpReg, scrReg); |
|
1697 |
} |
|
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1698 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1699 |
if (counters != NULL) { |
23491 | 1700 |
atomic_incl(ExternalAddress((address)counters->total_entry_count_addr()), scrReg); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1701 |
} |
51663 | 1702 |
|
1703 |
// Possible cases that we'll encounter in fast_lock |
|
1704 |
// ------------------------------------------------ |
|
1705 |
// * Inflated |
|
1706 |
// -- unlocked |
|
1707 |
// -- Locked |
|
1708 |
// = by self |
|
1709 |
// = by other |
|
1710 |
// * biased |
|
1711 |
// -- by Self |
|
1712 |
// -- by other |
|
1713 |
// * neutral |
|
1714 |
// * stack-locked |
|
1715 |
// -- by self |
|
1716 |
// = sp-proximity test hits |
|
1717 |
// = sp-proximity test generates false-negative |
|
1718 |
// -- by other |
|
1719 |
// |
|
1720 |
||
1721 |
Label IsInflated, DONE_LABEL; |
|
1722 |
||
1723 |
// it's stack-locked, biased or neutral |
|
1724 |
// TODO: optimize away redundant LDs of obj->mark and improve the markword triage |
|
1725 |
// order to reduce the number of conditional branches in the most common cases. |
|
1726 |
// Beware -- there's a subtle invariant that fetch of the markword |
|
1727 |
// at [FETCH], below, will never observe a biased encoding (*101b). |
|
1728 |
// If this invariant is not held we risk exclusion (safety) failure. |
|
1729 |
if (UseBiasedLocking && !UseOptoBiasInlining) { |
|
1730 |
biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, counters); |
|
1731 |
} |
|
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1732 |
|
23491 | 1733 |
#if INCLUDE_RTM_OPT |
51663 | 1734 |
if (UseRTMForStackLocks && use_rtm) { |
1735 |
rtm_stack_locking(objReg, tmpReg, scrReg, cx2Reg, |
|
1736 |
stack_rtm_counters, method_data, profile_rtm, |
|
1737 |
DONE_LABEL, IsInflated); |
|
1738 |
} |
|
23491 | 1739 |
#endif // INCLUDE_RTM_OPT |
1740 |
||
51663 | 1741 |
movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // [FETCH] |
57777
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1742 |
testptr(tmpReg, markWord::monitor_value); // inflated vs stack-locked|neutral|biased |
51663 | 1743 |
jccb(Assembler::notZero, IsInflated); |
1744 |
||
1745 |
// Attempt stack-locking ... |
|
57777
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1746 |
orptr (tmpReg, markWord::unlocked_value); |
51663 | 1747 |
movptr(Address(boxReg, 0), tmpReg); // Anticipate successful CAS |
51996
84743156e780
8188764: Obsolete AssumeMP and then remove all support for non-MP builds
dholmes
parents:
51976
diff
changeset
|
1748 |
lock(); |
51663 | 1749 |
cmpxchgptr(boxReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Updates tmpReg |
1750 |
if (counters != NULL) { |
|
1751 |
cond_inc32(Assembler::equal, |
|
1752 |
ExternalAddress((address)counters->fast_path_entry_count_addr())); |
|
1753 |
} |
|
1754 |
jcc(Assembler::equal, DONE_LABEL); // Success |
|
1755 |
||
1756 |
// Recursive locking. |
|
1757 |
// The object is stack-locked: markword contains stack pointer to BasicLock. |
|
1758 |
// Locked by current thread if difference with current SP is less than one page. |
|
1759 |
subptr(tmpReg, rsp); |
|
1760 |
// Next instruction set ZFlag == 1 (Success) if difference is less then one page. |
|
1761 |
andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - os::vm_page_size())) ); |
|
1762 |
movptr(Address(boxReg, 0), tmpReg); |
|
1763 |
if (counters != NULL) { |
|
1764 |
cond_inc32(Assembler::equal, |
|
1765 |
ExternalAddress((address)counters->fast_path_entry_count_addr())); |
|
1766 |
} |
|
1767 |
jmp(DONE_LABEL); |
|
1768 |
||
1769 |
bind(IsInflated); |
|
57777
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1770 |
// The object is inflated. tmpReg contains pointer to ObjectMonitor* + markWord::monitor_value |
23491 | 1771 |
|
1772 |
#if INCLUDE_RTM_OPT |
|
51663 | 1773 |
// Use the same RTM locking code in 32- and 64-bit VM. |
1774 |
if (use_rtm) { |
|
1775 |
rtm_inflated_locking(objReg, boxReg, tmpReg, scrReg, cx1Reg, cx2Reg, |
|
1776 |
rtm_counters, method_data, profile_rtm, DONE_LABEL); |
|
1777 |
} else { |
|
23491 | 1778 |
#endif // INCLUDE_RTM_OPT |
1779 |
||
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1780 |
#ifndef _LP64 |
51663 | 1781 |
// The object is inflated. |
1782 |
||
1783 |
// boxReg refers to the on-stack BasicLock in the current frame. |
|
1784 |
// We'd like to write: |
|
57777
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1785 |
// set box->_displaced_header = markWord::unused_mark(). Any non-0 value suffices. |
51663 | 1786 |
// This is convenient but results a ST-before-CAS penalty. The following CAS suffers |
1787 |
// additional latency as we have another ST in the store buffer that must drain. |
|
1788 |
||
1789 |
// avoid ST-before-CAS |
|
1790 |
// register juggle because we need tmpReg for cmpxchgptr below |
|
1791 |
movptr(scrReg, boxReg); |
|
1792 |
movptr(boxReg, tmpReg); // consider: LEA box, [tmp-2] |
|
1793 |
||
1794 |
// Optimistic form: consider XORL tmpReg,tmpReg |
|
1795 |
movptr(tmpReg, NULL_WORD); |
|
1796 |
||
1797 |
// Appears unlocked - try to swing _owner from null to non-null. |
|
1798 |
// Ideally, I'd manifest "Self" with get_thread and then attempt |
|
1799 |
// to CAS the register containing Self into m->Owner. |
|
1800 |
// But we don't have enough registers, so instead we can either try to CAS |
|
1801 |
// rsp or the address of the box (in scr) into &m->owner. If the CAS succeeds |
|
1802 |
// we later store "Self" into m->Owner. Transiently storing a stack address |
|
1803 |
// (rsp or the address of the box) into m->owner is harmless. |
|
1804 |
// Invariant: tmpReg == 0. tmpReg is EAX which is the implicit cmpxchg comparand. |
|
51996
84743156e780
8188764: Obsolete AssumeMP and then remove all support for non-MP builds
dholmes
parents:
51976
diff
changeset
|
1805 |
lock(); |
51663 | 1806 |
cmpxchgptr(scrReg, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); |
1807 |
movptr(Address(scrReg, 0), 3); // box->_displaced_header = 3 |
|
1808 |
// If we weren't able to swing _owner from NULL to the BasicLock |
|
1809 |
// then take the slow path. |
|
1810 |
jccb (Assembler::notZero, DONE_LABEL); |
|
1811 |
// update _owner from BasicLock to thread |
|
1812 |
get_thread (scrReg); // beware: clobbers ICCs |
|
1813 |
movptr(Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), scrReg); |
|
1814 |
xorptr(boxReg, boxReg); // set icc.ZFlag = 1 to indicate success |
|
1815 |
||
1816 |
// If the CAS fails we can either retry or pass control to the slow-path. |
|
1817 |
// We use the latter tactic. |
|
1818 |
// Pass the CAS result in the icc.ZFlag into DONE_LABEL |
|
1819 |
// If the CAS was successful ... |
|
1820 |
// Self has acquired the lock |
|
1821 |
// Invariant: m->_recursions should already be 0, so we don't need to explicitly set it. |
|
1822 |
// Intentional fall-through into DONE_LABEL ... |
|
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1823 |
#else // _LP64 |
51663 | 1824 |
// It's inflated |
1825 |
movq(scrReg, tmpReg); |
|
1826 |
xorq(tmpReg, tmpReg); |
|
1827 |
||
51996
84743156e780
8188764: Obsolete AssumeMP and then remove all support for non-MP builds
dholmes
parents:
51976
diff
changeset
|
1828 |
lock(); |
51663 | 1829 |
cmpxchgptr(r15_thread, Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); |
57777
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1830 |
// Unconditionally set box->_displaced_header = markWord::unused_mark(). |
51663 | 1831 |
// Without cast to int32_t movptr will destroy r10 which is typically obj. |
57777
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1832 |
movptr(Address(boxReg, 0), (int32_t)intptr_t(markWord::unused_mark().value())); |
51663 | 1833 |
// Intentional fall-through into DONE_LABEL ... |
1834 |
// Propagate ICC.ZF from CAS above into DONE_LABEL. |
|
23491 | 1835 |
#endif // _LP64 |
1836 |
#if INCLUDE_RTM_OPT |
|
51663 | 1837 |
} // use_rtm() |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1838 |
#endif |
51663 | 1839 |
// DONE_LABEL is a hot target - we'd really like to place it at the |
1840 |
// start of cache line by padding with NOPs. |
|
1841 |
// See the AMD and Intel software optimization manuals for the |
|
1842 |
// most efficient "long" NOP encodings. |
|
1843 |
// Unfortunately none of our alignment mechanisms suffice. |
|
1844 |
bind(DONE_LABEL); |
|
1845 |
||
1846 |
// At DONE_LABEL the icc ZFlag is set as follows ... |
|
1847 |
// Fast_Unlock uses the same protocol. |
|
1848 |
// ZFlag == 1 -> Success |
|
1849 |
// ZFlag == 0 -> Failure - force control through the slow-path |
|
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1850 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1851 |
|
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1852 |
// obj: object to unlock |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1853 |
// box: box address (displaced header location), killed. Must be EAX. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1854 |
// tmp: killed, cannot be obj nor box. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1855 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1856 |
// Some commentary on balanced locking: |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1857 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1858 |
// Fast_Lock and Fast_Unlock are emitted only for provably balanced lock sites. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1859 |
// Methods that don't have provably balanced locking are forced to run in the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1860 |
// interpreter - such methods won't be compiled to use fast_lock and fast_unlock. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1861 |
// The interpreter provides two properties: |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1862 |
// I1: At return-time the interpreter automatically and quietly unlocks any |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1863 |
// objects acquired the current activation (frame). Recall that the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1864 |
// interpreter maintains an on-stack list of locks currently held by |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1865 |
// a frame. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1866 |
// I2: If a method attempts to unlock an object that is not held by the |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1867 |
// the frame the interpreter throws IMSX. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1868 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1869 |
// Lets say A(), which has provably balanced locking, acquires O and then calls B(). |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1870 |
// B() doesn't have provably balanced locking so it runs in the interpreter. |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1871 |
// Control returns to A() and A() unlocks O. By I1 and I2, above, we know that O |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1872 |
// is still locked by A(). |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1873 |
// |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1874 |
// The only other source of unbalanced locking would be JNI. The "Java Native Interface: |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1875 |
// Programmer's Guide and Specification" claims that an object locked by jni_monitorenter |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1876 |
// should not be unlocked by "normal" java-level locking and vice-versa. The specification |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1877 |
// doesn't specify what will occur if a program engages in such mixed-mode locking, however. |
30244 | 1878 |
// Arguably given that the spec legislates the JNI case as undefined our implementation |
1879 |
// could reasonably *avoid* checking owner in Fast_Unlock(). |
|
1880 |
// In the interest of performance we elide m->Owner==Self check in unlock. |
|
1881 |
// A perfectly viable alternative is to elide the owner check except when |
|
1882 |
// Xcheck:jni is enabled. |
|
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1883 |
|
23491 | 1884 |
void MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register tmpReg, bool use_rtm) { |
1885 |
assert(boxReg == rax, ""); |
|
1886 |
assert_different_registers(objReg, boxReg, tmpReg); |
|
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1887 |
|
51663 | 1888 |
Label DONE_LABEL, Stacked, CheckSucc; |
1889 |
||
1890 |
// Critically, the biased locking test must have precedence over |
|
1891 |
// and appear before the (box->dhw == 0) recursive stack-lock test. |
|
1892 |
if (UseBiasedLocking && !UseOptoBiasInlining) { |
|
1893 |
biased_locking_exit(objReg, tmpReg, DONE_LABEL); |
|
1894 |
} |
|
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1895 |
|
23491 | 1896 |
#if INCLUDE_RTM_OPT |
51663 | 1897 |
if (UseRTMForStackLocks && use_rtm) { |
1898 |
assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking"); |
|
1899 |
Label L_regular_unlock; |
|
57777
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1900 |
movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // fetch markword |
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1901 |
andptr(tmpReg, markWord::biased_lock_mask_in_place); // look at 3 lock bits |
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1902 |
cmpptr(tmpReg, markWord::unlocked_value); // bits = 001 unlocked |
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1903 |
jccb(Assembler::notEqual, L_regular_unlock); // if !HLE RegularLock |
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1904 |
xend(); // otherwise end... |
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1905 |
jmp(DONE_LABEL); // ... and we're done |
51663 | 1906 |
bind(L_regular_unlock); |
1907 |
} |
|
23491 | 1908 |
#endif |
1909 |
||
57777
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1910 |
cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD); // Examine the displaced header |
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1911 |
jcc (Assembler::zero, DONE_LABEL); // 0 indicates recursive stack-lock |
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1912 |
movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Examine the object's markword |
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
55253
diff
changeset
|
1913 |
testptr(tmpReg, markWord::monitor_value); // Inflated? |
51663 | 1914 |
jccb (Assembler::zero, Stacked); |
1915 |
||
1916 |
// It's inflated. |
|
23491 | 1917 |
#if INCLUDE_RTM_OPT |
51663 | 1918 |
if (use_rtm) { |
1919 |
Label L_regular_inflated_unlock; |
|
1920 |
int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner); |
|
1921 |
movptr(boxReg, Address(tmpReg, owner_offset)); |
|
1922 |
testptr(boxReg, boxReg); |
|
1923 |
jccb(Assembler::notZero, L_regular_inflated_unlock); |
|
1924 |
xend(); |
|
1925 |
jmpb(DONE_LABEL); |
|
1926 |
bind(L_regular_inflated_unlock); |
|
1927 |
} |
|
23491 | 1928 |
#endif |
1929 |
||
51663 | 1930 |
// Despite our balanced locking property we still check that m->_owner == Self |
1931 |
// as java routines or native JNI code called by this thread might |
|
1932 |
// have released the lock. |
|
1933 |
// Refer to the comments in synchronizer.cpp for how we might encode extra |
|
1934 |
// state in _succ so we can avoid fetching EntryList|cxq. |
|
1935 |
// |
|
1936 |
// I'd like to add more cases in fast_lock() and fast_unlock() -- |
|
1937 |
// such as recursive enter and exit -- but we have to be wary of |
|
1938 |
// I$ bloat, T$ effects and BP$ effects. |
|
1939 |
// |
|
1940 |
// If there's no contention try a 1-0 exit. That is, exit without |
|
1941 |
// a costly MEMBAR or CAS. See synchronizer.cpp for details on how |
|
1942 |
// we detect and recover from the race that the 1-0 exit admits. |
|
1943 |
// |
|
1944 |
// Conceptually Fast_Unlock() must execute a STST|LDST "release" barrier |
|
1945 |
// before it STs null into _owner, releasing the lock. Updates |
|
1946 |
// to data protected by the critical section must be visible before |
|
1947 |
// we drop the lock (and thus before any other thread could acquire |
|
1948 |
// the lock and observe the fields protected by the lock). |
|
1949 |
// IA32's memory-model is SPO, so STs are ordered with respect to |
|
1950 |
// each other and there's no need for an explicit barrier (fence). |
|
1951 |
// See also http://gee.cs.oswego.edu/dl/jmm/cookbook.html. |
|
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1952 |
#ifndef _LP64 |
51663 | 1953 |
get_thread (boxReg); |
1954 |
||
1955 |
// Note that we could employ various encoding schemes to reduce |
|
1956 |
// the number of loads below (currently 4) to just 2 or 3. |
|
1957 |
// Refer to the comments in synchronizer.cpp. |
|
1958 |
// In practice the chain of fetches doesn't seem to impact performance, however. |
|
1959 |
xorptr(boxReg, boxReg); |
|
1960 |
orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions))); |
|
1961 |
jccb (Assembler::notZero, DONE_LABEL); |
|
1962 |
movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList))); |
|
1963 |
orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq))); |
|
1964 |
jccb (Assembler::notZero, CheckSucc); |
|
1965 |
movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD); |
|
1966 |
jmpb (DONE_LABEL); |
|
1967 |
||
1968 |
bind (Stacked); |
|
1969 |
// It's not inflated and it's not recursively stack-locked and it's not biased. |
|
1970 |
// It must be stack-locked. |
|
1971 |
// Try to reset the header to displaced header. |
|
1972 |
// The "box" value on the stack is stable, so we can reload |
|
1973 |
// and be assured we observe the same value as above. |
|
1974 |
movptr(tmpReg, Address(boxReg, 0)); |
|
51996
84743156e780
8188764: Obsolete AssumeMP and then remove all support for non-MP builds
dholmes
parents:
51976
diff
changeset
|
1975 |
lock(); |
51663 | 1976 |
cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box |
1977 |
// Intention fall-thru into DONE_LABEL |
|
1978 |
||
1979 |
// DONE_LABEL is a hot target - we'd really like to place it at the |
|
1980 |
// start of cache line by padding with NOPs. |
|
1981 |
// See the AMD and Intel software optimization manuals for the |
|
1982 |
// most efficient "long" NOP encodings. |
|
1983 |
// Unfortunately none of our alignment mechanisms suffice. |
|
1984 |
bind (CheckSucc); |
|
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
1985 |
#else // _LP64 |
51663 | 1986 |
// It's inflated |
1987 |
xorptr(boxReg, boxReg); |
|
1988 |
orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions))); |
|
1989 |
jccb (Assembler::notZero, DONE_LABEL); |
|
1990 |
movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq))); |
|
1991 |
orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList))); |
|
1992 |
jccb (Assembler::notZero, CheckSucc); |
|
1993 |
movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t)NULL_WORD); |
|
1994 |
jmpb (DONE_LABEL); |
|
1995 |
||
1996 |
// Try to avoid passing control into the slow_path ... |
|
1997 |
Label LSuccess, LGoSlowPath ; |
|
1998 |
bind (CheckSucc); |
|
1999 |
||
2000 |
// The following optional optimization can be elided if necessary |
|
2001 |
// Effectively: if (succ == null) goto SlowPath |
|
2002 |
// The code reduces the window for a race, however, |
|
2003 |
// and thus benefits performance. |
|
2004 |
cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), (int32_t)NULL_WORD); |
|
2005 |
jccb (Assembler::zero, LGoSlowPath); |
|
2006 |
||
2007 |
xorptr(boxReg, boxReg); |
|
2008 |
movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t)NULL_WORD); |
|
51996
84743156e780
8188764: Obsolete AssumeMP and then remove all support for non-MP builds
dholmes
parents:
51976
diff
changeset
|
2009 |
|
84743156e780
8188764: Obsolete AssumeMP and then remove all support for non-MP builds
dholmes
parents:
51976
diff
changeset
|
2010 |
// Memory barrier/fence |
84743156e780
8188764: Obsolete AssumeMP and then remove all support for non-MP builds
dholmes
parents:
51976
diff
changeset
|
2011 |
// Dekker pivot point -- fulcrum : ST Owner; MEMBAR; LD Succ |
84743156e780
8188764: Obsolete AssumeMP and then remove all support for non-MP builds
dholmes
parents:
51976
diff
changeset
|
2012 |
// Instead of MFENCE we use a dummy locked add of 0 to the top-of-stack. |
84743156e780
8188764: Obsolete AssumeMP and then remove all support for non-MP builds
dholmes
parents:
51976
diff
changeset
|
2013 |
// This is faster on Nehalem and AMD Shanghai/Barcelona. |
84743156e780
8188764: Obsolete AssumeMP and then remove all support for non-MP builds
dholmes
parents:
51976
diff
changeset
|
2014 |
// See https://blogs.oracle.com/dave/entry/instruction_selection_for_volatile_fences |
84743156e780
8188764: Obsolete AssumeMP and then remove all support for non-MP builds
dholmes
parents:
51976
diff
changeset
|
2015 |
// We might also restructure (ST Owner=0;barrier;LD _Succ) to |
84743156e780
8188764: Obsolete AssumeMP and then remove all support for non-MP builds
dholmes
parents:
51976
diff
changeset
|
2016 |
// (mov box,0; xchgq box, &m->Owner; LD _succ) . |
84743156e780
8188764: Obsolete AssumeMP and then remove all support for non-MP builds
dholmes
parents:
51976
diff
changeset
|
2017 |
lock(); addl(Address(rsp, 0), 0); |
84743156e780
8188764: Obsolete AssumeMP and then remove all support for non-MP builds
dholmes
parents:
51976
diff
changeset
|
2018 |
|
51663 | 2019 |
cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), (int32_t)NULL_WORD); |
2020 |
jccb (Assembler::notZero, LSuccess); |
|
2021 |
||
2022 |
// Rare inopportune interleaving - race. |
|
2023 |
// The successor vanished in the small window above. |
|
2024 |
// The lock is contended -- (cxq|EntryList) != null -- and there's no apparent successor. |
|
2025 |
// We need to ensure progress and succession. |
|
2026 |
// Try to reacquire the lock. |
|
2027 |
// If that fails then the new owner is responsible for succession and this |
|
2028 |
// thread needs to take no further action and can exit via the fast path (success). |
|
2029 |
// If the re-acquire succeeds then pass control into the slow path. |
|
2030 |
// As implemented, this latter mode is horrible because we generated more |
|
2031 |
// coherence traffic on the lock *and* artifically extended the critical section |
|
2032 |
// length while by virtue of passing control into the slow path. |
|
2033 |
||
2034 |
// box is really RAX -- the following CMPXCHG depends on that binding |
|
2035 |
// cmpxchg R,[M] is equivalent to rax = CAS(M,rax,R) |
|
51996
84743156e780
8188764: Obsolete AssumeMP and then remove all support for non-MP builds
dholmes
parents:
51976
diff
changeset
|
2036 |
lock(); |
51663 | 2037 |
cmpxchgptr(r15_thread, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); |
2038 |
// There's no successor so we tried to regrab the lock. |
|
2039 |
// If that didn't work, then another thread grabbed the |
|
2040 |
// lock so we're done (and exit was a success). |
|
2041 |
jccb (Assembler::notEqual, LSuccess); |
|
2042 |
// Intentional fall-through into slow-path |
|
2043 |
||
2044 |
bind (LGoSlowPath); |
|
2045 |
orl (boxReg, 1); // set ICC.ZF=0 to indicate failure |
|
2046 |
jmpb (DONE_LABEL); |
|
2047 |
||
2048 |
bind (LSuccess); |
|
2049 |
testl (boxReg, 0); // set ICC.ZF=1 to indicate success |
|
2050 |
jmpb (DONE_LABEL); |
|
2051 |
||
2052 |
bind (Stacked); |
|
2053 |
movptr(tmpReg, Address (boxReg, 0)); // re-fetch |
|
51996
84743156e780
8188764: Obsolete AssumeMP and then remove all support for non-MP builds
dholmes
parents:
51976
diff
changeset
|
2054 |
lock(); |
51663 | 2055 |
cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box |
2056 |
||
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2057 |
#endif |
51663 | 2058 |
bind(DONE_LABEL); |
22910
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2059 |
} |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2060 |
#endif // COMPILER2 |
88c3369b5967
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
kvn
parents:
21923
diff
changeset
|
2061 |
|
14626 | 2062 |
void MacroAssembler::c2bool(Register x) { |
2063 |
// implements x == 0 ? 0 : 1 |
|
2064 |
// note: must only look at least-significant byte of x |
|
2065 |
// since C-style booleans are stored in one byte |
|
2066 |
// only! (was bug) |
|
2067 |
andl(x, 0xFF); |
|
2068 |
setb(Assembler::notZero, x); |
|
2069 |
} |
|
2070 |
||
2071 |
// Wouldn't need if AddressLiteral version had new name |
|
2072 |
void MacroAssembler::call(Label& L, relocInfo::relocType rtype) { |
|
2073 |
Assembler::call(L, rtype); |
|
2074 |
} |
|
2075 |
||
2076 |
void MacroAssembler::call(Register entry) { |
|
2077 |
Assembler::call(entry); |
|
2078 |
} |
|
2079 |
||
2080 |
void MacroAssembler::call(AddressLiteral entry) { |
|
2081 |
if (reachable(entry)) { |
|
2082 |
Assembler::call_literal(entry.target(), entry.rspec()); |
|
2083 |
} else { |
|
2084 |
lea(rscratch1, entry); |
|
2085 |
Assembler::call(rscratch1); |
|
2086 |
} |
|
2087 |
} |
|
2088 |
||
35086
bbf32241d851
8072008: Emit direct call instead of linkTo* for recursive indy/MH.invoke* calls
vlivanov
parents:
34211
diff
changeset
|
2089 |
void MacroAssembler::ic_call(address entry, jint method_index) { |
bbf32241d851
8072008: Emit direct call instead of linkTo* for recursive indy/MH.invoke* calls
vlivanov
parents:
34211
diff
changeset
|
2090 |
RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index); |
14626 | 2091 |
movptr(rax, (intptr_t)Universe::non_oop_word()); |
2092 |
call(AddressLiteral(entry, rh)); |
|
2093 |
} |
|
2094 |
||
2095 |
// Implementation of call_VM versions |
|
2096 |
||
2097 |
void MacroAssembler::call_VM(Register oop_result, |
|
2098 |
address entry_point, |
|
2099 |
bool check_exceptions) { |
|
2100 |
Label C, E; |
|
2101 |
call(C, relocInfo::none); |
|
2102 |
jmp(E); |
|
2103 |
||
2104 |
bind(C); |
|
2105 |
call_VM_helper(oop_result, entry_point, 0, check_exceptions); |
|
2106 |
ret(0); |
|
2107 |
||
2108 |
bind(E); |
|
2109 |
} |
|
2110 |
||
2111 |
void MacroAssembler::call_VM(Register oop_result, |
|
2112 |
address entry_point, |
|
2113 |
Register arg_1, |
|
2114 |
bool check_exceptions) { |
|
2115 |
Label C, E; |
|
2116 |
call(C, relocInfo::none); |
|
2117 |
jmp(E); |
|
2118 |
||
2119 |
bind(C); |
|
2120 |
pass_arg1(this, arg_1); |
|
2121 |
call_VM_helper(oop_result, entry_point, 1, check_exceptions); |
|
2122 |
ret(0); |
|
2123 |
||
2124 |
bind(E); |
|
2125 |
} |
|
2126 |
||
2127 |
void MacroAssembler::call_VM(Register oop_result, |
|
2128 |
address entry_point, |
|
2129 |
Register arg_1, |
|
2130 |
Register arg_2, |
|
2131 |
bool check_exceptions) { |
|
2132 |
Label C, E; |
|
2133 |
call(C, relocInfo::none); |
|
2134 |
jmp(E); |
|
2135 |
||
2136 |
bind(C); |
|
2137 |
||
2138 |
LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); |
|
2139 |
||
2140 |
pass_arg2(this, arg_2); |
|
2141 |
pass_arg1(this, arg_1); |
|
2142 |
call_VM_helper(oop_result, entry_point, 2, check_exceptions); |
|
2143 |
ret(0); |
|
2144 |
||
2145 |
bind(E); |
|
2146 |
} |
|
2147 |
||
2148 |
void MacroAssembler::call_VM(Register oop_result, |
|
2149 |
address entry_point, |
|
2150 |
Register arg_1, |
|
2151 |
Register arg_2, |
|
2152 |
Register arg_3, |
|
2153 |
bool check_exceptions) { |
|
2154 |
Label C, E; |
|
2155 |
call(C, relocInfo::none); |
|
2156 |
jmp(E); |
|
2157 |
||
2158 |
bind(C); |
|
2159 |
||
2160 |
LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg")); |
|
2161 |
LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg")); |
|
2162 |
pass_arg3(this, arg_3); |
|
2163 |
||
2164 |
LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); |
|
2165 |
pass_arg2(this, arg_2); |
|
2166 |
||
2167 |
pass_arg1(this, arg_1); |
|
2168 |
call_VM_helper(oop_result, entry_point, 3, check_exceptions); |
|
2169 |
ret(0); |
|
2170 |
||
2171 |
bind(E); |
|
2172 |
} |
|
2173 |
||
2174 |
void MacroAssembler::call_VM(Register oop_result, |
|
2175 |
Register last_java_sp, |
|
2176 |
address entry_point, |
|
2177 |
int number_of_arguments, |
|
2178 |
bool check_exceptions) { |
|
2179 |
Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg); |
|
2180 |
call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions); |
|
2181 |
} |
|
2182 |
||
2183 |
void MacroAssembler::call_VM(Register oop_result, |
|
2184 |
Register last_java_sp, |
|
2185 |
address entry_point, |
|
2186 |
Register arg_1, |
|
2187 |
bool check_exceptions) { |
|
2188 |
pass_arg1(this, arg_1); |
|
2189 |
call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions); |
|
2190 |
} |
|
2191 |
||
2192 |
void MacroAssembler::call_VM(Register oop_result, |
|
2193 |
Register last_java_sp, |
|
2194 |
address entry_point, |
|
2195 |
Register arg_1, |
|
2196 |
Register arg_2, |
|
2197 |
bool check_exceptions) { |
|
2198 |
||
2199 |
LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); |
|
2200 |
pass_arg2(this, arg_2); |
|
2201 |
pass_arg1(this, arg_1); |
|
2202 |
call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions); |
|
2203 |
} |
|
2204 |
||
2205 |
void MacroAssembler::call_VM(Register oop_result, |
|
2206 |
Register last_java_sp, |
|
2207 |
address entry_point, |
|
2208 |
Register arg_1, |
|
2209 |
Register arg_2, |
|
2210 |
Register arg_3, |
|
2211 |
bool check_exceptions) { |
|
2212 |
LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg")); |
|
2213 |
LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg")); |
|
2214 |
pass_arg3(this, arg_3); |
|
2215 |
LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); |
|
2216 |
pass_arg2(this, arg_2); |
|
2217 |
pass_arg1(this, arg_1); |
|
2218 |
call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions); |
|
2219 |
} |
|
2220 |
||
2221 |
void MacroAssembler::super_call_VM(Register oop_result, |
|
2222 |
Register last_java_sp, |
|
2223 |
address entry_point, |
|
2224 |
int number_of_arguments, |
|
2225 |
bool check_exceptions) { |
|
2226 |
Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg); |
|
2227 |
MacroAssembler::call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions); |
|
2228 |
} |
|
2229 |
||
2230 |
void MacroAssembler::super_call_VM(Register oop_result, |
|
2231 |
Register last_java_sp, |
|
2232 |
address entry_point, |
|
2233 |
Register arg_1, |
|
2234 |
bool check_exceptions) { |
|
2235 |
pass_arg1(this, arg_1); |
|
2236 |
super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions); |
|
2237 |
} |
|
2238 |
||
2239 |
void MacroAssembler::super_call_VM(Register oop_result, |
|
2240 |
Register last_java_sp, |
|
2241 |
address entry_point, |
|
2242 |
Register arg_1, |
|
2243 |
Register arg_2, |
|
2244 |
bool check_exceptions) { |
|
2245 |
||
2246 |
LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); |
|
2247 |
pass_arg2(this, arg_2); |
|
2248 |
pass_arg1(this, arg_1); |
|
2249 |
super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions); |
|
2250 |
} |
|
2251 |
||
2252 |
void MacroAssembler::super_call_VM(Register oop_result, |
|
2253 |
Register last_java_sp, |
|
2254 |
address entry_point, |
|
2255 |
Register arg_1, |
|
2256 |
Register arg_2, |
|
2257 |
Register arg_3, |
|
2258 |
bool check_exceptions) { |
|
2259 |
LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg")); |
|
2260 |
LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg")); |
|
2261 |
pass_arg3(this, arg_3); |
|
2262 |
LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); |
|
2263 |
pass_arg2(this, arg_2); |
|
2264 |
pass_arg1(this, arg_1); |
|
2265 |
super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions); |
|
2266 |
} |
|
2267 |
||
2268 |
void MacroAssembler::call_VM_base(Register oop_result, |
|
2269 |
Register java_thread, |
|
2270 |
Register last_java_sp, |
|
2271 |
address entry_point, |
|
2272 |
int number_of_arguments, |
|
2273 |
bool check_exceptions) { |
|
2274 |
// determine java_thread register |
|
2275 |
if (!java_thread->is_valid()) { |
|
2276 |
#ifdef _LP64 |
|
2277 |
java_thread = r15_thread; |
|
2278 |
#else |
|
2279 |
java_thread = rdi; |
|
2280 |
get_thread(java_thread); |
|
2281 |
#endif // LP64 |
|
2282 |
} |
|
2283 |
// determine last_java_sp register |
|
2284 |
if (!last_java_sp->is_valid()) { |
|
2285 |
last_java_sp = rsp; |
|
2286 |
} |
|
2287 |
// debugging support |
|
2288 |
assert(number_of_arguments >= 0 , "cannot have negative number of arguments"); |
|
2289 |
LP64_ONLY(assert(java_thread == r15_thread, "unexpected register")); |
|
2290 |
#ifdef ASSERT |
|
2291 |
// TraceBytecodes does not use r12 but saves it over the call, so don't verify |
|
2292 |
// r12 is the heapbase. |
|
19979
ebe1dbb6e1aa
8015107: NPG: Use consistent naming for metaspace concepts
ehelin
parents:
19319
diff
changeset
|
2293 |
LP64_ONLY(if ((UseCompressedOops || UseCompressedClassPointers) && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");) |
14626 | 2294 |
#endif // ASSERT |
2295 |
||
2296 |
assert(java_thread != oop_result , "cannot use the same register for java_thread & oop_result"); |
|
2297 |
assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp"); |
|
2298 |
||
2299 |
// push java thread (becomes first argument of C function) |
|
2300 |
||
2301 |
NOT_LP64(push(java_thread); number_of_arguments++); |
|
2302 |
LP64_ONLY(mov(c_rarg0, r15_thread)); |
|
2303 |
||
2304 |
// set last Java frame before call |
|
2305 |
assert(last_java_sp != rbp, "can't use ebp/rbp"); |
|
2306 |
||
2307 |
// Only interpreter should have to set fp |
|
2308 |
set_last_Java_frame(java_thread, last_java_sp, rbp, NULL); |
|
2309 |
||
2310 |
// do the call, remove parameters |
|
2311 |
MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments); |
|
2312 |
||
2313 |
// restore the thread (cannot use the pushed argument since arguments |
|
2314 |
// may be overwritten by C code generated by an optimizing compiler); |
|
2315 |
// however can use the register value directly if it is callee saved. |
|
2316 |
if (LP64_ONLY(true ||) java_thread == rdi || java_thread == rsi) { |
|
2317 |
// rdi & rsi (also r15) are callee saved -> nothing to do |
|
2318 |
#ifdef ASSERT |
|
2319 |
guarantee(java_thread != rax, "change this code"); |
|
2320 |
push(rax); |
|
2321 |
{ Label L; |
|
2322 |
get_thread(rax); |
|
2323 |
cmpptr(java_thread, rax); |
|
2324 |
jcc(Assembler::equal, L); |
|
2325 |
STOP("MacroAssembler::call_VM_base: rdi not callee saved?"); |
|
2326 |
bind(L); |
|
2327 |
} |
|
2328 |
pop(rax); |
|
2329 |
#endif |
|
2330 |
} else { |
|
2331 |
get_thread(java_thread); |
|
2332 |
} |
|
2333 |
// reset last Java frame |
|
2334 |
// Only interpreter should have to clear fp |
|
40644
39e631ed7145
8161598: Kitchensink fails: assert(nm->insts_contains(original_pc)) failed: original PC must be in nmethod/CompiledMethod
dlong
parents:
39256
diff
changeset
|
2335 |
reset_last_Java_frame(java_thread, true); |
14626 | 2336 |
|
2337 |
// C++ interp handles this in the interpreter |
|
2338 |
check_and_handle_popframe(java_thread); |
|
2339 |
check_and_handle_earlyret(java_thread); |
|
2340 |
||
2341 |
if (check_exceptions) { |
|
2342 |
// check for pending exceptions (java_thread is set upon return) |
|
2343 |
cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t) NULL_WORD); |
|
2344 |
#ifndef _LP64 |
|
2345 |
jump_cc(Assembler::notEqual, |
|
2346 |
RuntimeAddress(StubRoutines::forward_exception_entry())); |
|
2347 |
#else |
|
2348 |
// This used to conditionally jump to forward_exception however it is |
|
2349 |
// possible if we relocate that the branch will not reach. So we must jump |
|
2350 |
// around so we can always reach |
|
2351 |
||
2352 |
Label ok; |
|
2353 |
jcc(Assembler::equal, ok); |
|
2354 |
jump(RuntimeAddress(StubRoutines::forward_exception_entry())); |
|
2355 |
bind(ok); |
|
2356 |
#endif // LP64 |
|
2357 |
} |
|
2358 |
||
2359 |
// get oop result if there is one and reset the value in the thread |
|
2360 |
if (oop_result->is_valid()) { |
|
2361 |
get_vm_result(oop_result, java_thread); |
|
2362 |
} |
|
2363 |
} |
|
2364 |
||
2365 |
void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) { |
|
2366 |
||
2367 |
// Calculate the value for last_Java_sp |
|
2368 |
// somewhat subtle. call_VM does an intermediate call |
|
2369 |
// which places a return address on the stack just under the |
|
2370 |
// stack pointer as the user finsihed with it. This allows |
|
2371 |
// use to retrieve last_Java_pc from last_Java_sp[-1]. |
|
2372 |
// On 32bit we then have to push additional args on the stack to accomplish |
|
2373 |
// the actual requested call. On 64bit call_VM only can use register args |
|
2374 |
// so the only extra space is the return address that call_VM created. |
|
2375 |
// This hopefully explains the calculations here. |
|
2376 |
||
2377 |
#ifdef _LP64 |
|
2378 |
// We've pushed one address, correct last_Java_sp |
|
2379 |
lea(rax, Address(rsp, wordSize)); |
|
2380 |
#else |
|
2381 |
lea(rax, Address(rsp, (1 + number_of_arguments) * wordSize)); |
|
2382 |
#endif // LP64 |
|
2383 |
||
2384 |
call_VM_base(oop_result, noreg, rax, entry_point, number_of_arguments, check_exceptions); |
|
2385 |
||
2386 |
} |
|
2387 |
||
38699
f8bec5f6b09c
8154473: Update for CompilerDirectives to control stub generation and intrinsics
vdeshpande
parents:
38241
diff
changeset
|
2388 |
// Use this method when MacroAssembler version of call_VM_leaf_base() should be called from Interpreter. |
f8bec5f6b09c
8154473: Update for CompilerDirectives to control stub generation and intrinsics
vdeshpande
parents:
38241
diff
changeset
|
2389 |
void MacroAssembler::call_VM_leaf0(address entry_point) { |
f8bec5f6b09c
8154473: Update for CompilerDirectives to control stub generation and intrinsics
vdeshpande
parents:
38241
diff
changeset
|
2390 |
MacroAssembler::call_VM_leaf_base(entry_point, 0); |
f8bec5f6b09c
8154473: Update for CompilerDirectives to control stub generation and intrinsics
vdeshpande
parents:
38241
diff
changeset
|
2391 |
} |
f8bec5f6b09c
8154473: Update for CompilerDirectives to control stub generation and intrinsics
vdeshpande
parents:
38241
diff
changeset
|
2392 |
|
14626 | 2393 |
void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) { |
2394 |
call_VM_leaf_base(entry_point, number_of_arguments); |
|
2395 |
} |
|
2396 |
||
2397 |
void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) { |
|
2398 |
pass_arg0(this, arg_0); |
|
2399 |
call_VM_leaf(entry_point, 1); |
|
2400 |
} |
|
2401 |
||
2402 |
void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) { |
|
2403 |
||
2404 |
LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg")); |
|
2405 |
pass_arg1(this, arg_1); |
|
2406 |
pass_arg0(this, arg_0); |
|
2407 |
call_VM_leaf(entry_point, 2); |
|
2408 |
} |
|
2409 |
||
2410 |
void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) { |
|
2411 |
LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg")); |
|
2412 |
LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); |
|
2413 |
pass_arg2(this, arg_2); |
|
2414 |
LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg")); |
|
2415 |
pass_arg1(this, arg_1); |
|
2416 |
pass_arg0(this, arg_0); |
|
2417 |
call_VM_leaf(entry_point, 3); |
|
2418 |
} |
|
2419 |
||
2420 |
void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) { |
|
2421 |
pass_arg0(this, arg_0); |
|
2422 |
MacroAssembler::call_VM_leaf_base(entry_point, 1); |
|
2423 |
} |
|
2424 |
||
2425 |
void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) { |
|
2426 |
||
2427 |
LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg")); |
|
2428 |
pass_arg1(this, arg_1); |
|
2429 |
pass_arg0(this, arg_0); |
|
2430 |
MacroAssembler::call_VM_leaf_base(entry_point, 2); |
|
2431 |
} |
|
2432 |
||
2433 |
void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) { |
|
2434 |
LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg")); |
|
2435 |
LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); |
|
2436 |
pass_arg2(this, arg_2); |
|
2437 |
LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg")); |
|
2438 |
pass_arg1(this, arg_1); |
|
2439 |
pass_arg0(this, arg_0); |
|
2440 |
MacroAssembler::call_VM_leaf_base(entry_point, 3); |
|
2441 |
} |
|
2442 |
||
2443 |
void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) { |
|
2444 |
LP64_ONLY(assert(arg_0 != c_rarg3, "smashed arg")); |
|
2445 |
LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg")); |
|
2446 |
LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg")); |
|
2447 |
pass_arg3(this, arg_3); |
|
2448 |
LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg")); |
|
2449 |
LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); |
|
2450 |
pass_arg2(this, arg_2); |
|
2451 |
LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg")); |
|
2452 |
pass_arg1(this, arg_1); |
|
2453 |
pass_arg0(this, arg_0); |
|
2454 |
MacroAssembler::call_VM_leaf_base(entry_point, 4); |
|
2455 |
} |
|
2456 |
||
2457 |
void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) { |
|
2458 |
movptr(oop_result, Address(java_thread, JavaThread::vm_result_offset())); |
|
2459 |
movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD); |
|
2460 |
verify_oop(oop_result, "broken oop in call_VM_base"); |
|
2461 |
} |
|
2462 |
||
2463 |
void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) { |
|
2464 |
movptr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset())); |
|
2465 |
movptr(Address(java_thread, JavaThread::vm_result_2_offset()), NULL_WORD); |
|
2466 |
} |
|
2467 |
||
2468 |
void MacroAssembler::check_and_handle_earlyret(Register java_thread) { |
|
2469 |
} |
|
2470 |
||
2471 |
void MacroAssembler::check_and_handle_popframe(Register java_thread) { |
|
2472 |
} |
|
2473 |
||
2474 |
void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm) { |
|
2475 |
if (reachable(src1)) { |
|
2476 |
cmpl(as_Address(src1), imm); |
|
2477 |
} else { |
|
2478 |
lea(rscratch1, src1); |
|
2479 |
cmpl(Address(rscratch1, 0), imm); |
|
2480 |
} |
|
2481 |
} |
|
2482 |
||
2483 |
void MacroAssembler::cmp32(Register src1, AddressLiteral src2) { |
|
2484 |
assert(!src2.is_lval(), "use cmpptr"); |
|
2485 |
if (reachable(src2)) { |
|
2486 |
cmpl(src1, as_Address(src2)); |
|
2487 |
} else { |
|
2488 |
lea(rscratch1, src2); |
|
2489 |
cmpl(src1, Address(rscratch1, 0)); |
|
2490 |
} |
|
2491 |
} |
|
2492 |
||
2493 |
void MacroAssembler::cmp32(Register src1, int32_t imm) { |
|
2494 |
Assembler::cmpl(src1, imm); |
|
2495 |
} |
|
2496 |
||
2497 |
void MacroAssembler::cmp32(Register src1, Address src2) { |
|
2498 |
Assembler::cmpl(src1, src2); |
|
2499 |
} |
|
2500 |
||
2501 |
void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) { |
|
2502 |
ucomisd(opr1, opr2); |
|
2503 |
||
2504 |
Label L; |
|
2505 |
if (unordered_is_less) { |
|
2506 |
movl(dst, -1); |
|
2507 |
jcc(Assembler::parity, L); |
|
2508 |
jcc(Assembler::below , L); |
|
2509 |
movl(dst, 0); |
|
2510 |
jcc(Assembler::equal , L); |
|
2511 |
increment(dst); |
|
2512 |
} else { // unordered is greater |
|
2513 |
movl(dst, 1); |
|
2514 |
jcc(Assembler::parity, L); |
|
2515 |
jcc(Assembler::above , L); |
|
2516 |
movl(dst, 0); |
|
2517 |
jcc(Assembler::equal , L); |
|
2518 |
decrementl(dst); |
|
2519 |
} |
|
2520 |
bind(L); |
|
2521 |
} |
|
2522 |
||
2523 |
void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) { |
|
2524 |
ucomiss(opr1, opr2); |
|
2525 |
||
2526 |
Label L; |
|
2527 |
if (unordered_is_less) { |
|
2528 |
movl(dst, -1); |
|
2529 |
jcc(Assembler::parity, L); |
|
2530 |
jcc(Assembler::below , L); |
|
2531 |
movl(dst, 0); |
|
2532 |
jcc(Assembler::equal , L); |
|
2533 |
increment(dst); |
|
2534 |
} else { // unordered is greater |
|
2535 |
movl(dst, 1); |
|
2536 |
jcc(Assembler::parity, L); |
|
2537 |
jcc(Assembler::above , L); |
|
2538 |
movl(dst, 0); |
|
2539 |
jcc(Assembler::equal , L); |
|
2540 |
decrementl(dst); |
|
2541 |
} |
|
2542 |
bind(L); |
|
2543 |
} |
|
2544 |
||
2545 |
||
2546 |
void MacroAssembler::cmp8(AddressLiteral src1, int imm) { |
|
2547 |
if (reachable(src1)) { |
|
2548 |
cmpb(as_Address(src1), imm); |
|
2549 |
} else { |
|
2550 |
lea(rscratch1, src1); |
|
2551 |
cmpb(Address(rscratch1, 0), imm); |
|
2552 |
} |
|
2553 |
} |
|
2554 |
||
2555 |
void MacroAssembler::cmpptr(Register src1, AddressLiteral src2) { |
|
2556 |
#ifdef _LP64 |
|
2557 |
if (src2.is_lval()) { |
|
2558 |
movptr(rscratch1, src2); |
|
2559 |
Assembler::cmpq(src1, rscratch1); |
|
2560 |
} else if (reachable(src2)) { |
|
2561 |
cmpq(src1, as_Address(src2)); |
|
2562 |
} else { |
|
2563 |
lea(rscratch1, src2); |
|
2564 |
Assembler::cmpq(src1, Address(rscratch1, 0)); |
|
2565 |
} |
|
2566 |
#else |
|
2567 |
if (src2.is_lval()) { |
|
2568 |
cmp_literal32(src1, (int32_t) src2.target(), src2.rspec()); |
|
2569 |
} else { |
|
2570 |
cmpl(src1, as_Address(src2)); |
|
2571 |
} |
|
2572 |
#endif // _LP64 |
|
2573 |
} |
|
2574 |
||
2575 |
void MacroAssembler::cmpptr(Address src1, AddressLiteral src2) { |
|
2576 |
assert(src2.is_lval(), "not a mem-mem compare"); |
|
2577 |
#ifdef _LP64 |
|
2578 |
// moves src2's literal address |
|
2579 |
movptr(rscratch1, src2); |
|
2580 |
Assembler::cmpq(src1, rscratch1); |
|
2581 |
#else |
|
2582 |
cmp_literal32(src1, (int32_t) src2.target(), src2.rspec()); |
|
2583 |
#endif // _LP64 |
|
2584 |
} |
|
2585 |
||
47683
f433d49aceb4
8184914: Use MacroAssembler::cmpoop() consistently when comparing heap objects
rkennke
parents:
47580
diff
changeset
|
2586 |
void MacroAssembler::cmpoop(Register src1, Register src2) { |
50536
8434981a4137
8203157: Object equals abstraction for BarrierSetAssembler
rkennke
parents:
50534
diff
changeset
|
2587 |
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); |
8434981a4137
8203157: Object equals abstraction for BarrierSetAssembler
rkennke
parents:
50534
diff
changeset
|
2588 |
bs->obj_equals(this, src1, src2); |
47683
f433d49aceb4
8184914: Use MacroAssembler::cmpoop() consistently when comparing heap objects
rkennke
parents:
47580
diff
changeset
|
2589 |
} |
f433d49aceb4
8184914: Use MacroAssembler::cmpoop() consistently when comparing heap objects
rkennke
parents:
47580
diff
changeset
|
2590 |
|
f433d49aceb4
8184914: Use MacroAssembler::cmpoop() consistently when comparing heap objects
rkennke
parents:
47580
diff
changeset
|
2591 |
void MacroAssembler::cmpoop(Register src1, Address src2) { |
50536
8434981a4137
8203157: Object equals abstraction for BarrierSetAssembler
rkennke
parents:
50534
diff
changeset
|
2592 |
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); |
8434981a4137
8203157: Object equals abstraction for BarrierSetAssembler
rkennke
parents:
50534
diff
changeset
|
2593 |
bs->obj_equals(this, src1, src2); |
47683
f433d49aceb4
8184914: Use MacroAssembler::cmpoop() consistently when comparing heap objects
rkennke
parents:
47580
diff
changeset
|
2594 |
} |
f433d49aceb4
8184914: Use MacroAssembler::cmpoop() consistently when comparing heap objects
rkennke
parents:
47580
diff
changeset
|
2595 |
|
f433d49aceb4
8184914: Use MacroAssembler::cmpoop() consistently when comparing heap objects
rkennke
parents:
47580
diff
changeset
|
2596 |
#ifdef _LP64 |
f433d49aceb4
8184914: Use MacroAssembler::cmpoop() consistently when comparing heap objects
rkennke
parents:
47580
diff
changeset
|
2597 |
void MacroAssembler::cmpoop(Register src1, jobject src2) { |
f433d49aceb4
8184914: Use MacroAssembler::cmpoop() consistently when comparing heap objects
rkennke
parents:
47580
diff
changeset
|
2598 |
movoop(rscratch1, src2); |
50536
8434981a4137
8203157: Object equals abstraction for BarrierSetAssembler
rkennke
parents:
50534
diff
changeset
|
2599 |
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); |
8434981a4137
8203157: Object equals abstraction for BarrierSetAssembler
rkennke
parents:
50534
diff
changeset
|
2600 |
bs->obj_equals(this, src1, rscratch1); |
47683
f433d49aceb4
8184914: Use MacroAssembler::cmpoop() consistently when comparing heap objects
rkennke
parents:
47580
diff
changeset
|
2601 |
} |
f433d49aceb4
8184914: Use MacroAssembler::cmpoop() consistently when comparing heap objects
rkennke
parents:
47580
diff
changeset
|
2602 |
#endif |
f433d49aceb4
8184914: Use MacroAssembler::cmpoop() consistently when comparing heap objects
rkennke
parents:
47580
diff
changeset
|
2603 |
|
14626 | 2604 |
void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr) { |
2605 |
if (reachable(adr)) { |
|
51996
84743156e780
8188764: Obsolete AssumeMP and then remove all support for non-MP builds
dholmes
parents:
51976
diff
changeset
|
2606 |
lock(); |
14626 | 2607 |
cmpxchgptr(reg, as_Address(adr)); |
2608 |
} else { |
|
2609 |
lea(rscratch1, adr); |
|
51996
84743156e780
8188764: Obsolete AssumeMP and then remove all support for non-MP builds
dholmes
parents:
51976
diff
changeset
|
2610 |
lock(); |
14626 | 2611 |
cmpxchgptr(reg, Address(rscratch1, 0)); |
2612 |
} |
|
2613 |
} |
|
2614 |
||
2615 |
void MacroAssembler::cmpxchgptr(Register reg, Address adr) { |
|
2616 |
LP64_ONLY(cmpxchgq(reg, adr)) NOT_LP64(cmpxchgl(reg, adr)); |
|
2617 |
} |
|
2618 |
||
2619 |
void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src) { |
|
2620 |
if (reachable(src)) { |
|
2621 |
Assembler::comisd(dst, as_Address(src)); |
|
2622 |
} else { |
|
2623 |
lea(rscratch1, src); |
|
2624 |
Assembler::comisd(dst, Address(rscratch1, 0)); |
|
2625 |
} |
|
2626 |
} |
|
2627 |
||
2628 |
void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src) { |
|
2629 |
if (reachable(src)) { |
|
2630 |
Assembler::comiss(dst, as_Address(src)); |
|
2631 |
} else { |
|
2632 |
lea(rscratch1, src); |
|
2633 |
Assembler::comiss(dst, Address(rscratch1, 0)); |
|
2634 |
} |
|
2635 |
} |
|
2636 |
||
2637 |
||
2638 |
void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr) { |
|
2639 |
Condition negated_cond = negate_condition(cond); |
|
2640 |
Label L; |
|
2641 |
jcc(negated_cond, L); |
|
23491 | 2642 |
pushf(); // Preserve flags |
14626 | 2643 |
atomic_incl(counter_addr); |
23491 | 2644 |
popf(); |
14626 | 2645 |
bind(L); |
2646 |
} |
|
2647 |
||
2648 |
int MacroAssembler::corrected_idivl(Register reg) { |
|
2649 |
// Full implementation of Java idiv and irem; checks for |
|
2650 |
// special case as described in JVM spec., p.243 & p.271. |
|
2651 |
// The function returns the (pc) offset of the idivl |
|
2652 |
// instruction - may be needed for implicit exceptions. |
|
2653 |
// |
|
2654 |
// normal case special case |
|
2655 |
// |
|
2656 |
// input : rax,: dividend min_int |
|
2657 |
// reg: divisor (may not be rax,/rdx) -1 |
|
2658 |
// |
|
2659 |
// output: rax,: quotient (= rax, idiv reg) min_int |
|
2660 |
// rdx: remainder (= rax, irem reg) 0 |
|
2661 |
assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register"); |
|
2662 |
const int min_int = 0x80000000; |
|
2663 |
Label normal_case, special_case; |
|
2664 |
||
2665 |
// check for special case |
|
2666 |
cmpl(rax, min_int); |
|
2667 |
jcc(Assembler::notEqual, normal_case); |
|
2668 |
xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0) |
|
2669 |
cmpl(reg, -1); |
|
2670 |
jcc(Assembler::equal, special_case); |
|
2671 |
||
2672 |
// handle normal case |
|
2673 |
bind(normal_case); |
|
2674 |
cdql(); |
|
2675 |
int idivl_offset = offset(); |
|
2676 |
idivl(reg); |
|
2677 |
||
2678 |
// normal and special case exit |
|
2679 |
bind(special_case); |
|
2680 |
||
2681 |
return idivl_offset; |
|
2682 |
} |
|
2683 |
||
2684 |
||
2685 |
||
2686 |
void MacroAssembler::decrementl(Register reg, int value) { |
|
2687 |
if (value == min_jint) {subl(reg, value) ; return; } |
|
2688 |
if (value < 0) { incrementl(reg, -value); return; } |
|
2689 |
if (value == 0) { ; return; } |
|
2690 |
if (value == 1 && UseIncDec) { decl(reg) ; return; } |
|
2691 |
/* else */ { subl(reg, value) ; return; } |
|
2692 |
} |
|
2693 |
||
2694 |
void MacroAssembler::decrementl(Address dst, int value) { |
|
2695 |
if (value == min_jint) {subl(dst, value) ; return; } |
|
2696 |
if (value < 0) { incrementl(dst, -value); return; } |
|
2697 |
if (value == 0) { ; return; } |
|
2698 |
if (value == 1 && UseIncDec) { decl(dst) ; return; } |
|
2699 |
/* else */ { subl(dst, value) ; return; } |
|
2700 |
} |
|
2701 |
||
2702 |
void MacroAssembler::division_with_shift (Register reg, int shift_value) { |
|
2703 |
assert (shift_value > 0, "illegal shift value"); |
|
2704 |
Label _is_positive; |
|
2705 |
testl (reg, reg); |
|
2706 |
jcc (Assembler::positive, _is_positive); |
|
2707 |
int offset = (1 << shift_value) - 1 ; |
|
2708 |
||
2709 |
if (offset == 1) { |
|
2710 |
incrementl(reg); |
|
2711 |
} else { |
|
2712 |
addl(reg, offset); |
|
2713 |
} |
|
2714 |
||
2715 |
bind (_is_positive); |
|
2716 |
sarl(reg, shift_value); |
|
2717 |
} |
|
2718 |
||
2719 |
void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src) { |
|
2720 |
if (reachable(src)) { |
|
2721 |
Assembler::divsd(dst, as_Address(src)); |
|
2722 |
} else { |
|
2723 |
lea(rscratch1, src); |
|
2724 |
Assembler::divsd(dst, Address(rscratch1, 0)); |
|
2725 |
} |
|
2726 |
} |
|
2727 |
||
2728 |
void MacroAssembler::divss(XMMRegister dst, AddressLiteral src) { |
|
2729 |
if (reachable(src)) { |
|
2730 |
Assembler::divss(dst, as_Address(src)); |
|
2731 |
} else { |
|
2732 |
lea(rscratch1, src); |
|
2733 |
Assembler::divss(dst, Address(rscratch1, 0)); |
|
2734 |
} |
|
2735 |
} |
|
2736 |
||
2737 |
// !defined(COMPILER2) is because of stupid core builds |
|
33160
c59f1676d27e
8136421: JEP 243: Java-Level JVM Compiler Interface
twisti
parents:
33089
diff
changeset
|
2738 |
#if !defined(_LP64) || defined(COMPILER1) || !defined(COMPILER2) || INCLUDE_JVMCI |
14626 | 2739 |
void MacroAssembler::empty_FPU_stack() { |
2740 |
if (VM_Version::supports_mmx()) { |
|
2741 |
emms(); |
|
2742 |
} else { |
|
2743 |
for (int i = 8; i-- > 0; ) ffree(i); |
|
2744 |
} |
|
2745 |
} |
|
33160
c59f1676d27e
8136421: JEP 243: Java-Level JVM Compiler Interface
twisti
parents:
33089
diff
changeset
|
2746 |
#endif // !LP64 || C1 || !C2 || INCLUDE_JVMCI |
14626 | 2747 |
|
2748 |
||
2749 |
void MacroAssembler::enter() { |
|
2750 |
push(rbp); |
|
2751 |
mov(rbp, rsp); |
|
2752 |
} |
|
2753 |
||
2754 |
// A 5 byte nop that is safe for patching (see patch_verified_entry) |
|
2755 |
void MacroAssembler::fat_nop() { |
|
2756 |
if (UseAddressNop) { |
|
2757 |
addr_nop_5(); |
|
2758 |
} else { |
|
14837
a75c3082d106
8004250: replace AbstractAssembler a_byte/a_long with emit_int8/emit_int32
twisti
parents:
14834
diff
changeset
|
2759 |
emit_int8(0x26); // es: |
a75c3082d106
8004250: replace AbstractAssembler a_byte/a_long with emit_int8/emit_int32
twisti
parents:
14834
diff
changeset
|
2760 |
emit_int8(0x2e); // cs: |
a75c3082d106
8004250: replace AbstractAssembler a_byte/a_long with emit_int8/emit_int32
twisti
parents:
14834
diff
changeset
|
2761 |
emit_int8(0x64); // fs: |
a75c3082d106
8004250: replace AbstractAssembler a_byte/a_long with emit_int8/emit_int32
twisti
parents:
14834
diff
changeset
|
2762 |
emit_int8(0x65); // gs: |
a75c3082d106
8004250: replace AbstractAssembler a_byte/a_long with emit_int8/emit_int32
twisti
parents:
14834
diff
changeset
|
2763 |
emit_int8((unsigned char)0x90); |
14626 | 2764 |
} |
2765 |
} |
|
2766 |
||
2767 |
void MacroAssembler::fcmp(Register tmp) { |
|
2768 |
fcmp(tmp, 1, true, true); |
|
2769 |
} |
|
2770 |
||
2771 |
void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) { |
|
2772 |
assert(!pop_right || pop_left, "usage error"); |
|
2773 |
if (VM_Version::supports_cmov()) { |
|
2774 |
assert(tmp == noreg, "unneeded temp"); |
|
2775 |
if (pop_left) { |
|
2776 |
fucomip(index); |
|
2777 |
} else { |
|
2778 |
fucomi(index); |
|
2779 |
} |
|
2780 |
if (pop_right) { |
|
2781 |
fpop(); |
|
2782 |
} |
|
2783 |
} else { |
|
2784 |
assert(tmp != noreg, "need temp"); |
|
2785 |
if (pop_left) { |
|
2786 |
if (pop_right) { |
|
2787 |
fcompp(); |
|
2788 |
} else { |
|
2789 |
fcomp(index); |
|
2790 |
} |
|
2791 |
} else { |
|
2792 |
fcom(index); |
|
2793 |
} |
|
2794 |
// convert FPU condition into eflags condition via rax, |
|
2795 |
save_rax(tmp); |
|
2796 |
fwait(); fnstsw_ax(); |
|
2797 |
sahf(); |
|
2798 |
restore_rax(tmp); |
|
2799 |
} |
|
2800 |
// condition codes set as follows: |
|
2801 |
// |
|
2802 |
// CF (corresponds to C0) if x < y |
|
2803 |
// PF (corresponds to C2) if unordered |
|
2804 |
// ZF (corresponds to C3) if x = y |
|
2805 |
} |
|
2806 |
||
2807 |
void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less) { |
|
2808 |
fcmp2int(dst, unordered_is_less, 1, true, true); |
|
2809 |
} |
|
2810 |
||
2811 |
void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right) { |
|
2812 |
fcmp(VM_Version::supports_cmov() ? noreg : dst, index, pop_left, pop_right); |
|
2813 |
Label L; |
|
2814 |
if (unordered_is_less) { |
|
2815 |
movl(dst, -1); |
|
2816 |
jcc(Assembler::parity, L); |
|
2817 |
jcc(Assembler::below , L); |
|
2818 |
movl(dst, 0); |
|
2819 |
jcc(Assembler::equal , L); |
|
2820 |
increment(dst); |
|
2821 |
} else { // unordered is greater |
|
2822 |
movl(dst, 1); |
|
2823 |
jcc(Assembler::parity, L); |
|
2824 |
jcc(Assembler::above , L); |
|
2825 |
movl(dst, 0); |
|
2826 |
jcc(Assembler::equal , L); |
|
2827 |
decrementl(dst); |
|
2828 |
} |
|
2829 |
bind(L); |
|
2830 |
} |
|
2831 |
||
2832 |
void MacroAssembler::fld_d(AddressLiteral src) { |
|
2833 |
fld_d(as_Address(src)); |
|
2834 |
} |
|
2835 |
||
2836 |
void MacroAssembler::fld_s(AddressLiteral src) { |
|
2837 |
fld_s(as_Address(src)); |
|
2838 |
} |
|
2839 |
||
2840 |
void MacroAssembler::fld_x(AddressLiteral src) { |
|
2841 |
Assembler::fld_x(as_Address(src)); |
|
2842 |
} |
|
2843 |
||
2844 |
void MacroAssembler::fldcw(AddressLiteral src) { |
|
2845 |
Assembler::fldcw(as_Address(src)); |
|
2846 |
} |
|
2847 |
||
33089 | 2848 |
void MacroAssembler::mulpd(XMMRegister dst, AddressLiteral src) { |
2849 |
if (reachable(src)) { |
|
2850 |
Assembler::mulpd(dst, as_Address(src)); |
|
2851 |
} else { |
|
2852 |
lea(rscratch1, src); |
|
2853 |
Assembler::mulpd(dst, Address(rscratch1, 0)); |
|
2854 |
} |
|
2855 |
} |
|
2856 |
||
14626 | 2857 |
void MacroAssembler::increase_precision() { |
2858 |
subptr(rsp, BytesPerWord); |
|
2859 |
fnstcw(Address(rsp, 0)); |
|
2860 |
movl(rax, Address(rsp, 0)); |
|
2861 |
orl(rax, 0x300); |
|
2862 |
push(rax); |
|
2863 |
fldcw(Address(rsp, 0)); |
|
2864 |
pop(rax); |
|
2865 |
} |
|
2866 |
||
2867 |
void MacroAssembler::restore_precision() { |
|
2868 |
fldcw(Address(rsp, 0)); |
|
2869 |
addptr(rsp, BytesPerWord); |
|
2870 |
} |
|
2871 |
||
2872 |
void MacroAssembler::fpop() { |
|
2873 |
ffree(); |
|
2874 |
fincstp(); |
|
2875 |
} |
|
2876 |
||
32391
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2877 |
void MacroAssembler::load_float(Address src) { |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2878 |
if (UseSSE >= 1) { |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2879 |
movflt(xmm0, src); |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2880 |
} else { |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2881 |
LP64_ONLY(ShouldNotReachHere()); |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2882 |
NOT_LP64(fld_s(src)); |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2883 |
} |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2884 |
} |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2885 |
|
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2886 |
void MacroAssembler::store_float(Address dst) { |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2887 |
if (UseSSE >= 1) { |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2888 |
movflt(dst, xmm0); |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2889 |
} else { |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2890 |
LP64_ONLY(ShouldNotReachHere()); |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2891 |
NOT_LP64(fstp_s(dst)); |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2892 |
} |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2893 |
} |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2894 |
|
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2895 |
void MacroAssembler::load_double(Address src) { |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2896 |
if (UseSSE >= 2) { |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2897 |
movdbl(xmm0, src); |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2898 |
} else { |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2899 |
LP64_ONLY(ShouldNotReachHere()); |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2900 |
NOT_LP64(fld_d(src)); |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2901 |
} |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2902 |
} |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2903 |
|
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2904 |
void MacroAssembler::store_double(Address dst) { |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2905 |
if (UseSSE >= 2) { |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2906 |
movdbl(dst, xmm0); |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2907 |
} else { |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2908 |
LP64_ONLY(ShouldNotReachHere()); |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2909 |
NOT_LP64(fstp_d(dst)); |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2910 |
} |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2911 |
} |
01e2f5e916c7
8076373: In 32-bit VM interpreter and compiled code process NaN values differently
zmajo
parents:
32203
diff
changeset
|
2912 |
|
14626 | 2913 |
void MacroAssembler::fremr(Register tmp) { |
2914 |
save_rax(tmp); |
|
2915 |
{ Label L; |
|
2916 |
bind(L); |
|
2917 |
fprem(); |
|
2918 |
fwait(); fnstsw_ax(); |
|
2919 |
#ifdef _LP64 |
|
2920 |
testl(rax, 0x400); |
|
2921 |
jcc(Assembler::notEqual, L); |
|
2922 |
#else |
|
2923 |
sahf(); |
|
2924 |
jcc(Assembler::parity, L); |
|
2925 |
#endif // _LP64 |
|
2926 |
} |
|
2927 |
restore_rax(tmp); |
|
2928 |
// Result is in ST0. |
|
2929 |
// Note: fxch & fpop to get rid of ST1 |
|
2930 |
// (otherwise FPU stack could overflow eventually) |
|
2931 |
fxch(1); |
|
2932 |
fpop(); |
|
2933 |
} |
|
2934 |
||
41323 | 2935 |
// dst = c = a * b + c |
2936 |
void MacroAssembler::fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) { |
|
2937 |
Assembler::vfmadd231sd(c, a, b); |
|
2938 |
if (dst != c) { |
|
2939 |
movdbl(dst, c); |
|
2940 |
} |
|
2941 |
} |
|
2942 |
||
2943 |
// dst = c = a * b + c |
|
2944 |
void MacroAssembler::fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) { |
|
2945 |
Assembler::vfmadd231ss(c, a, b); |
|
2946 |
if (dst != c) { |
|
2947 |
movflt(dst, c); |
|
2948 |
} |
|
2949 |
} |
|
2950 |
||
46528 | 2951 |
// dst = c = a * b + c |
2952 |
void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) { |
|
2953 |
Assembler::vfmadd231pd(c, a, b, vector_len); |
|
2954 |
if (dst != c) { |
|
2955 |
vmovdqu(dst, c); |
|
2956 |
} |
|
2957 |
} |
|
2958 |
||
2959 |
// dst = c = a * b + c |
|
2960 |
void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) { |
|
2961 |
Assembler::vfmadd231ps(c, a, b, vector_len); |
|
2962 |
if (dst != c) { |
|
2963 |
vmovdqu(dst, c); |
|
2964 |
} |
|
2965 |
} |
|
2966 |
||
2967 |
// dst = c = a * b + c |
|
2968 |
void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) { |
|
2969 |
Assembler::vfmadd231pd(c, a, b, vector_len); |
|
2970 |
if (dst != c) { |
|
2971 |
vmovdqu(dst, c); |
|
2972 |
} |
|
2973 |
} |
|
2974 |
||
2975 |
// dst = c = a * b + c |
|
2976 |
void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) { |
|
2977 |
Assembler::vfmadd231ps(c, a, b, vector_len); |
|
2978 |
if (dst != c) { |
|
2979 |
vmovdqu(dst, c); |
|
2980 |
} |
|
2981 |
} |
|
14626 | 2982 |
|
2983 |
void MacroAssembler::incrementl(AddressLiteral dst) { |
|
2984 |
if (reachable(dst)) { |
|
2985 |
incrementl(as_Address(dst)); |
|
2986 |
} else { |
|
2987 |
lea(rscratch1, dst); |
|
2988 |
incrementl(Address(rscratch1, 0)); |
|
2989 |
} |
|
2990 |
} |
|
2991 |
||
2992 |
void MacroAssembler::incrementl(ArrayAddress dst) { |
|
2993 |
incrementl(as_Address(dst)); |
|
2994 |
} |
|
2995 |
||
2996 |
void MacroAssembler::incrementl(Register reg, int value) { |
|
2997 |
if (value == min_jint) {addl(reg, value) ; return; } |
|
2998 |
if (value < 0) { decrementl(reg, -value); return; } |
|
2999 |
if (value == 0) { ; return; } |
|
3000 |
if (value == 1 && UseIncDec) { incl(reg) ; return; } |
|
3001 |
/* else */ { addl(reg, value) ; return; } |
|
3002 |
} |
|
3003 |
||
3004 |
void MacroAssembler::incrementl(Address dst, int value) { |
|
3005 |
if (value == min_jint) {addl(dst, value) ; return; } |
|
3006 |
if (value < 0) { decrementl(dst, -value); return; } |
|
3007 |
if (value == 0) { ; return; } |
|
3008 |
if (value == 1 && UseIncDec) { incl(dst) ; return; } |
|
3009 |
/* else */ { addl(dst, value) ; return; } |
|
3010 |
} |
|
3011 |
||
3012 |
void MacroAssembler::jump(AddressLiteral dst) { |
|
3013 |
if (reachable(dst)) { |
|
3014 |
jmp_literal(dst.target(), dst.rspec()); |
|
3015 |
} else { |
|
3016 |
lea(rscratch1, dst); |
|
3017 |
jmp(rscratch1); |
|
3018 |
} |
|
3019 |
} |
|
3020 |
||
3021 |
void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst) { |
|
3022 |
if (reachable(dst)) { |
|
3023 |
InstructionMark im(this); |
|
3024 |
relocate(dst.reloc()); |
|
3025 |
const int short_size = 2; |
|
3026 |
const int long_size = 6; |
|
3027 |
int offs = (intptr_t)dst.target() - ((intptr_t)pc()); |
|
3028 |
if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) { |
|
3029 |
// 0111 tttn #8-bit disp |
|
14837
a75c3082d106
8004250: replace AbstractAssembler a_byte/a_long with emit_int8/emit_int32
twisti
parents:
14834
diff
changeset
|
3030 |
emit_int8(0x70 | cc); |
a75c3082d106
8004250: replace AbstractAssembler a_byte/a_long with emit_int8/emit_int32
twisti
parents:
14834
diff
changeset
|
3031 |
emit_int8((offs - short_size) & 0xFF); |
14626 | 3032 |
} else { |
3033 |
// 0000 1111 1000 tttn #32-bit disp |
|
14837
a75c3082d106
8004250: replace AbstractAssembler a_byte/a_long with emit_int8/emit_int32
twisti
parents:
14834
diff
changeset
|
3034 |
emit_int8(0x0F); |
a75c3082d106
8004250: replace AbstractAssembler a_byte/a_long with emit_int8/emit_int32
twisti
parents:
14834
diff
changeset
|
3035 |
emit_int8((unsigned char)(0x80 | cc)); |
15116
af423dcb739c
8004537: replace AbstractAssembler emit_long with emit_int32
twisti
parents:
15115
diff
changeset
|
3036 |
emit_int32(offs - long_size); |
14626 | 3037 |
} |
3038 |
} else { |
|
3039 |
#ifdef ASSERT |
|
3040 |
warning("reversing conditional branch"); |
|
3041 |
#endif /* ASSERT */ |
|
3042 |
Label skip; |
|
3043 |
jccb(reverse[cc], skip); |
|
3044 |
lea(rscratch1, dst); |
|
3045 |
Assembler::jmp(rscratch1); |
|
3046 |
bind(skip); |
|
3047 |
} |
|
3048 |
} |
|
3049 |
||
3050 |
void MacroAssembler::ldmxcsr(AddressLiteral src) { |
|
3051 |
if (reachable(src)) { |
|
3052 |
Assembler::ldmxcsr(as_Address(src)); |
|
3053 |
} else { |
|
3054 |
lea(rscratch1, src); |
|
3055 |
Assembler::ldmxcsr(Address(rscratch1, 0)); |
|
3056 |
} |
|
3057 |
} |
|
3058 |
||
3059 |
int MacroAssembler::load_signed_byte(Register dst, Address src) { |
|
3060 |
int off; |
|
3061 |
if (LP64_ONLY(true ||) VM_Version::is_P6()) { |
|
3062 |
off = offset(); |
|
3063 |
movsbl(dst, src); // movsxb |
|
3064 |
} else { |
|
3065 |
off = load_unsigned_byte(dst, src); |
|
3066 |
shll(dst, 24); |
|
3067 |
sarl(dst, 24); |
|
3068 |
} |
|
3069 |
return off; |
|
3070 |
} |
|
3071 |
||
3072 |
// Note: load_signed_short used to be called load_signed_word. |
|
3073 |
// Although the 'w' in x86 opcodes refers to the term "word" in the assembler |
|
3074 |
// manual, which means 16 bits, that usage is found nowhere in HotSpot code. |
|
3075 |
// The term "word" in HotSpot means a 32- or 64-bit machine word. |
|
3076 |
int MacroAssembler::load_signed_short(Register dst, Address src) { |
|
3077 |
int off; |
|
3078 |
if (LP64_ONLY(true ||) VM_Version::is_P6()) { |
|
3079 |
// This is dubious to me since it seems safe to do a signed 16 => 64 bit |
|
3080 |
// version but this is what 64bit has always done. This seems to imply |
|
3081 |
// that users are only using 32bits worth. |
|
3082 |
off = offset(); |
|
3083 |
movswl(dst, src); // movsxw |
|
3084 |
} else { |
|
3085 |
off = load_unsigned_short(dst, src); |
|
3086 |
shll(dst, 16); |
|
3087 |
sarl(dst, 16); |
|
3088 |
} |
|
3089 |
return off; |
|
3090 |
} |
|
3091 |
||
3092 |
int MacroAssembler::load_unsigned_byte(Register dst, Address src) { |
|
3093 |
// According to Intel Doc. AP-526, "Zero-Extension of Short", p.16, |
|
3094 |
// and "3.9 Partial Register Penalties", p. 22). |
|
3095 |
int off; |
|
3096 |
if (LP64_ONLY(true || ) VM_Version::is_P6() || src.uses(dst)) { |
|
3097 |
off = offset(); |
|
3098 |
movzbl(dst, src); // movzxb |
|
3099 |
} else { |
|
3100 |
xorl(dst, dst); |
|
3101 |
off = offset(); |
|
3102 |
movb(dst, src); |
|
3103 |
} |
|
3104 |
return off; |
|
3105 |
} |
|
3106 |
||
3107 |
// Note: load_unsigned_short used to be called load_unsigned_word. |
|
3108 |
int MacroAssembler::load_unsigned_short(Register dst, Address src) { |
|
3109 |
// According to Intel Doc. AP-526, "Zero-Extension of Short", p.16, |
|
3110 |
// and "3.9 Partial Register Penalties", p. 22). |
|
3111 |
int off; |
|
3112 |
if (LP64_ONLY(true ||) VM_Version::is_P6() || src.uses(dst)) { |
|
3113 |
off = offset(); |
|
3114 |
movzwl(dst, src); // movzxw |
|
3115 |
} else { |
|
3116 |
xorl(dst, dst); |
|
3117 |
off = offset(); |
|
3118 |
movw(dst, src); |
|
3119 |
} |
|
3120 |
return off; |
|
3121 |
} |
|
3122 |
||
3123 |
void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) { |
|
3124 |
switch (size_in_bytes) { |
|
3125 |
#ifndef _LP64 |
|
3126 |
case 8: |
|
3127 |
assert(dst2 != noreg, "second dest register required"); |
|
3128 |
movl(dst, src); |
|
3129 |
movl(dst2, src.plus_disp(BytesPerInt)); |
|
3130 |
break; |
|
3131 |
#else |
|
3132 |
case 8: movq(dst, src); break; |
|
3133 |
#endif |
|
3134 |
case 4: movl(dst, src); break; |
|
3135 |
case 2: is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break; |
|
3136 |
case 1: is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break; |
|
3137 |
default: ShouldNotReachHere(); |
|
3138 |
} |
|
3139 |
} |
|
3140 |
||
3141 |
void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) { |
|
3142 |
switch (size_in_bytes) { |
|
3143 |
#ifndef _LP64 |
|
3144 |
case 8: |
|
3145 |
assert(src2 != noreg, "second source register required"); |
|
3146 |
movl(dst, src); |
|
3147 |
movl(dst.plus_disp(BytesPerInt), src2); |
|
3148 |
break; |
|
3149 |
#else |
|
3150 |
case 8: movq(dst, src); break; |
|
3151 |
#endif |
|
3152 |
case 4: movl(dst, src); break; |
|
3153 |
case 2: movw(dst, src); break; |
|
3154 |
case 1: movb(dst, src); break; |
|
3155 |
default: ShouldNotReachHere(); |
|
3156 |
} |
|
3157 |
} |
|
3158 |
||
3159 |
void MacroAssembler::mov32(AddressLiteral dst, Register src) { |
|
3160 |
if (reachable(dst)) { |
|
3161 |
movl(as_Address(dst), src); |
|
3162 |
} else { |
|
3163 |
lea(rscratch1, dst); |
|
3164 |
movl(Address(rscratch1, 0), src); |
|
3165 |
} |
|
3166 |
} |
|
3167 |
||
3168 |
void MacroAssembler::mov32(Register dst, AddressLiteral src) { |
|
3169 |
if (reachable(src)) { |
|
3170 |
movl(dst, as_Address(src)); |
|
3171 |
} else { |
|
3172 |
lea(rscratch1, src); |
|
3173 |
movl(dst, Address(rscratch1, 0)); |
|
3174 |
} |
|
3175 |
} |
|
3176 |
||
3177 |
// C++ bool manipulation |
|
3178 |
||
3179 |
void MacroAssembler::movbool(Register dst, Address src) { |
|
3180 |
if(sizeof(bool) == 1) |
|
3181 |
movb(dst, src); |
|
3182 |
else if(sizeof(bool) == 2) |
|
3183 |
movw(dst, src); |
|
3184 |
else if(sizeof(bool) == 4) |
|
3185 |
movl(dst, src); |
|
3186 |
else |
|
3187 |
// unsupported |
|
3188 |
ShouldNotReachHere(); |
|
3189 |
} |
|
3190 |
||
3191 |
void MacroAssembler::movbool(Address dst, bool boolconst) { |
|
3192 |
if(sizeof(bool) == 1) |
|
3193 |
movb(dst, (int) boolconst); |
|
3194 |
else if(sizeof(bool) == 2) |
|
3195 |
movw(dst, (int) boolconst); |
|
3196 |
else if(sizeof(bool) == 4) |
|
3197 |
movl(dst, (int) boolconst); |
|
3198 |
else |
|
3199 |
// unsupported |
|
3200 |
ShouldNotReachHere(); |
|
3201 |
} |
|
3202 |
||
3203 |
void MacroAssembler::movbool(Address dst, Register src) { |
|
3204 |
if(sizeof(bool) == 1) |
|
3205 |
movb(dst, src); |
|
3206 |
else if(sizeof(bool) == 2) |
|
3207 |
movw(dst, src); |
|
3208 |
else if(sizeof(bool) == 4) |
|
3209 |
movl(dst, src); |
|
3210 |
else |
|
3211 |
// unsupported |
|
3212 |
ShouldNotReachHere(); |
|
3213 |
} |
|
3214 |
||
3215 |
void MacroAssembler::movbyte(ArrayAddress dst, int src) { |
|
3216 |
movb(as_Address(dst), src); |
|
3217 |
} |
|
3218 |
||
3219 |
void MacroAssembler::movdl(XMMRegister dst, AddressLiteral src) { |
|
3220 |
if (reachable(src)) { |
|
3221 |
movdl(dst, as_Address(src)); |
|
3222 |
} else { |
|
3223 |
lea(rscratch1, src); |
|
3224 |
movdl(dst, Address(rscratch1, 0)); |
|
3225 |
} |
|
3226 |
} |
|
3227 |
||
3228 |
void MacroAssembler::movq(XMMRegister dst, AddressLiteral src) { |
|
3229 |
if (reachable(src)) { |
|
3230 |
movq(dst, as_Address(src)); |
|
3231 |
} else { |
|
3232 |
lea(rscratch1, src); |
|
3233 |
movq(dst, Address(rscratch1, 0)); |
|
3234 |
} |
|
3235 |
} |
|
3236 |
||
52003
be4614f04eb6
8211375: Minimal VM build failures after JDK-8211251 (Default mask register for avx512 instructions)
shade
parents:
51996
diff
changeset
|
3237 |
#ifdef COMPILER2 |
38049 | 3238 |
void MacroAssembler::setvectmask(Register dst, Register src) { |
52003
be4614f04eb6
8211375: Minimal VM build failures after JDK-8211251 (Default mask register for avx512 instructions)
shade
parents:
51996
diff
changeset
|
3239 |
guarantee(PostLoopMultiversioning, "must be"); |
38049 | 3240 |
Assembler::movl(dst, 1); |
3241 |
Assembler::shlxl(dst, dst, src); |
|
3242 |
Assembler::decl(dst); |
|
3243 |
Assembler::kmovdl(k1, dst); |
|
3244 |
Assembler::movl(dst, src); |
|
3245 |
} |
|
3246 |
||
3247 |
void MacroAssembler::restorevectmask() { |
|
52003
be4614f04eb6
8211375: Minimal VM build failures after JDK-8211251 (Default mask register for avx512 instructions)
shade
parents:
51996
diff
changeset
|
3248 |
guarantee(PostLoopMultiversioning, "must be"); |
38049 | 3249 |
Assembler::knotwl(k1, k0); |
3250 |
} |
|
52003
be4614f04eb6
8211375: Minimal VM build failures after JDK-8211251 (Default mask register for avx512 instructions)
shade
parents:
51996
diff
changeset
|
3251 |
#endif // COMPILER2 |
38049 | 3252 |
|
14626 | 3253 |
void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src) { |
3254 |
if (reachable(src)) { |
|
3255 |
if (UseXmmLoadAndClearUpper) { |
|
3256 |
movsd (dst, as_Address(src)); |
|
3257 |
} else { |
|
3258 |
movlpd(dst, as_Address(src)); |
|
3259 |
} |
|
3260 |
} else { |
|
3261 |
lea(rscratch1, src); |
|
3262 |
if (UseXmmLoadAndClearUpper) { |
|
3263 |
movsd (dst, Address(rscratch1, 0)); |
|
3264 |
} else { |
|
3265 |
movlpd(dst, Address(rscratch1, 0)); |
|
3266 |
} |
|
3267 |
} |
|
3268 |
} |
|
3269 |
||
3270 |
void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src) { |
|
3271 |
if (reachable(src)) { |
|
3272 |
movss(dst, as_Address(src)); |
|
3273 |
} else { |
|
3274 |
lea(rscratch1, src); |
|
3275 |
movss(dst, Address(rscratch1, 0)); |
|
3276 |
} |
|
3277 |
} |
|
3278 |
||
3279 |
void MacroAssembler::movptr(Register dst, Register src) { |
|
3280 |
LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src)); |
|
3281 |
} |
|
3282 |
||
3283 |
void MacroAssembler::movptr(Register dst, Address src) { |
|
3284 |
LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src)); |
|
3285 |
} |
|
3286 |
||
3287 |
// src should NEVER be a real pointer. Use AddressLiteral for true pointers |
|
3288 |
void MacroAssembler::movptr(Register dst, intptr_t src) { |
|
3289 |
LP64_ONLY(mov64(dst, src)) NOT_LP64(movl(dst, src)); |
|
3290 |
} |
|
3291 |
||
3292 |
void MacroAssembler::movptr(Address dst, Register src) { |
|
3293 |
LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src)); |
|
3294 |
} |
|
3295 |
||
34162 | 3296 |
void MacroAssembler::movdqu(Address dst, XMMRegister src) { |
51857 | 3297 |
assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); |
34162 | 3298 |
Assembler::movdqu(dst, src); |
3299 |
} |
|
3300 |
||
3301 |
void MacroAssembler::movdqu(XMMRegister dst, Address src) { |
|
51857 | 3302 |
assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); |
34162 | 3303 |
Assembler::movdqu(dst, src); |
3304 |
} |
|
3305 |
||
3306 |
void MacroAssembler::movdqu(XMMRegister dst, XMMRegister src) { |
|
51857 | 3307 |
assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); |
34162 | 3308 |
Assembler::movdqu(dst, src); |
3309 |
} |
|
3310 |
||
43423
bcaab17f72a5
8171974: Fix for R10 Register clobbering with usage of ExternalAddress
vdeshpande
parents:
42587
diff
changeset
|
3311 |
void MacroAssembler::movdqu(XMMRegister dst, AddressLiteral src, Register scratchReg) { |
14626 | 3312 |
if (reachable(src)) { |
34162 | 3313 |
movdqu(dst, as_Address(src)); |
14626 | 3314 |
} else { |
43423
bcaab17f72a5
8171974: Fix for R10 Register clobbering with usage of ExternalAddress
vdeshpande
parents:
42587
diff
changeset
|
3315 |
lea(scratchReg, src); |
bcaab17f72a5
8171974: Fix for R10 Register clobbering with usage of ExternalAddress
vdeshpande
parents:
42587
diff
changeset
|
3316 |
movdqu(dst, Address(scratchReg, 0)); |
34162 | 3317 |
} |
3318 |
} |
|
3319 |
||
3320 |
void MacroAssembler::vmovdqu(Address dst, XMMRegister src) { |
|
51857 | 3321 |
assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); |
34162 | 3322 |
Assembler::vmovdqu(dst, src); |
3323 |
} |
|
3324 |
||
3325 |
void MacroAssembler::vmovdqu(XMMRegister dst, Address src) { |
|
51857 | 3326 |
assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); |
34162 | 3327 |
Assembler::vmovdqu(dst, src); |
3328 |
} |
|
3329 |
||
3330 |
void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) { |
|
51857 | 3331 |
assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); |
34162 | 3332 |
Assembler::vmovdqu(dst, src); |
3333 |
} |
|
3334 |
||
54750 | 3335 |
void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, Register scratch_reg) { |
34162 | 3336 |
if (reachable(src)) { |
3337 |
vmovdqu(dst, as_Address(src)); |
|
3338 |
} |
|
3339 |
else { |
|
54750 | 3340 |
lea(scratch_reg, src); |
3341 |
vmovdqu(dst, Address(scratch_reg, 0)); |
|
14626 | 3342 |
} |
3343 |
} |
|
3344 |
||
50860
480a96a43b62
8205528: Base64 encoding algorithm using AVX512 instructions
kvn
parents:
50803
diff
changeset
|
3345 |
void MacroAssembler::evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { |
480a96a43b62
8205528: Base64 encoding algorithm using AVX512 instructions
kvn
parents:
50803
diff
changeset
|
3346 |
if (reachable(src)) { |
480a96a43b62
8205528: Base64 encoding algorithm using AVX512 instructions
kvn
parents:
50803
diff
changeset
|
3347 |
Assembler::evmovdquq(dst, as_Address(src), vector_len); |
480a96a43b62
8205528: Base64 encoding algorithm using AVX512 instructions
kvn
parents:
50803
diff
changeset
|
3348 |
} else { |
480a96a43b62
8205528: Base64 encoding algorithm using AVX512 instructions
kvn
parents:
50803
diff
changeset
|
3349 |
lea(rscratch, src); |
480a96a43b62
8205528: Base64 encoding algorithm using AVX512 instructions
kvn
parents:
50803
diff
changeset
|
3350 |
Assembler::evmovdquq(dst, Address(rscratch, 0), vector_len); |
480a96a43b62
8205528: Base64 encoding algorithm using AVX512 instructions
kvn
parents:
50803
diff
changeset
|
3351 |
} |
480a96a43b62
8205528: Base64 encoding algorithm using AVX512 instructions
kvn
parents:
50803
diff
changeset
|
3352 |
} |
480a96a43b62
8205528: Base64 encoding algorithm using AVX512 instructions
kvn
parents:
50803
diff
changeset
|
3353 |
|
18507
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
3354 |
void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src) { |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
3355 |
if (reachable(src)) { |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
3356 |
Assembler::movdqa(dst, as_Address(src)); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
3357 |
} else { |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
3358 |
lea(rscratch1, src); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
3359 |
Assembler::movdqa(dst, Address(rscratch1, 0)); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
3360 |
} |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
3361 |
} |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
3362 |
|
14626 | 3363 |
void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src) { |
3364 |
if (reachable(src)) { |
|
3365 |
Assembler::movsd(dst, as_Address(src)); |
|
3366 |
} else { |
|
3367 |
lea(rscratch1, src); |
|
3368 |
Assembler::movsd(dst, Address(rscratch1, 0)); |
|
3369 |
} |
|
3370 |
} |
|
3371 |
||
3372 |
void MacroAssembler::movss(XMMRegister dst, AddressLiteral src) { |
|
3373 |
if (reachable(src)) { |
|
3374 |
Assembler::movss(dst, as_Address(src)); |
|
3375 |
} else { |
|
3376 |
lea(rscratch1, src); |
|
3377 |
Assembler::movss(dst, Address(rscratch1, 0)); |
|
3378 |
} |
|
3379 |
} |
|
3380 |
||
3381 |
void MacroAssembler::mulsd(XMMRegister dst, AddressLiteral src) { |
|
3382 |
if (reachable(src)) { |
|
3383 |
Assembler::mulsd(dst, as_Address(src)); |
|
3384 |
} else { |
|
3385 |
lea(rscratch1, src); |
|
3386 |
Assembler::mulsd(dst, Address(rscratch1, 0)); |
|
3387 |
} |
|
3388 |
} |
|
3389 |
||
3390 |
void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src) { |
|
3391 |
if (reachable(src)) { |
|
3392 |
Assembler::mulss(dst, as_Address(src)); |
|
3393 |
} else { |
|
3394 |
lea(rscratch1, src); |
|
3395 |
Assembler::mulss(dst, Address(rscratch1, 0)); |
|
3396 |
} |
|
3397 |
} |
|
3398 |
||
3399 |
void MacroAssembler::null_check(Register reg, int offset) { |
|
3400 |
if (needs_explicit_null_check(offset)) { |
|
3401 |
// provoke OS NULL exception if reg = NULL by |
|
3402 |
// accessing M[reg] w/o changing any (non-CC) registers |
|
3403 |
// NOTE: cmpl is plenty here to provoke a segv |
|
3404 |
cmpptr(rax, Address(reg, 0)); |
|
3405 |
// Note: should probably use testl(rax, Address(reg, 0)); |
|
3406 |
// may be shorter code (however, this version of |
|
3407 |
// testl needs to be implemented first) |
|
3408 |
} else { |
|
3409 |
// nothing to do, (later) access of M[reg + offset] |
|
3410 |
// will provoke OS NULL exception if reg = NULL |
|
3411 |
} |
|
3412 |
} |
|
3413 |
||
3414 |
void MacroAssembler::os_breakpoint() { |
|
3415 |
// instead of directly emitting a breakpoint, call os:breakpoint for better debugability |
|
3416 |
// (e.g., MSVC can't call ps() otherwise) |
|
3417 |
call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint))); |
|
3418 |
} |
|
3419 |
||
46560
388aa8d67c80
8181449: Fix debug.hpp / globalDefinitions.hpp dependency inversion
kbarrett
parents:
46530
diff
changeset
|
3420 |
void MacroAssembler::unimplemented(const char* what) { |
48968
8c64b94dca9d
8197608: MacroAssembler::unimplemented calls global operator new[]
thartmann
parents:
48826
diff
changeset
|
3421 |
const char* buf = NULL; |
8c64b94dca9d
8197608: MacroAssembler::unimplemented calls global operator new[]
thartmann
parents:
48826
diff
changeset
|
3422 |
{ |
8c64b94dca9d
8197608: MacroAssembler::unimplemented calls global operator new[]
thartmann
parents:
48826
diff
changeset
|
3423 |
ResourceMark rm; |
8c64b94dca9d
8197608: MacroAssembler::unimplemented calls global operator new[]
thartmann
parents:
48826
diff
changeset
|
3424 |
stringStream ss; |
8c64b94dca9d
8197608: MacroAssembler::unimplemented calls global operator new[]
thartmann
parents:
48826
diff
changeset
|
3425 |
ss.print("unimplemented: %s", what); |
8c64b94dca9d
8197608: MacroAssembler::unimplemented calls global operator new[]
thartmann
parents:
48826
diff
changeset
|
3426 |
buf = code_string(ss.as_string()); |
8c64b94dca9d
8197608: MacroAssembler::unimplemented calls global operator new[]
thartmann
parents:
48826
diff
changeset
|
3427 |
} |
8c64b94dca9d
8197608: MacroAssembler::unimplemented calls global operator new[]
thartmann
parents:
48826
diff
changeset
|
3428 |
stop(buf); |
46560
388aa8d67c80
8181449: Fix debug.hpp / globalDefinitions.hpp dependency inversion
kbarrett
parents:
46530
diff
changeset
|
3429 |
} |
388aa8d67c80
8181449: Fix debug.hpp / globalDefinitions.hpp dependency inversion
kbarrett
parents:
46530
diff
changeset
|
3430 |
|
34162 | 3431 |
#ifdef _LP64 |
3432 |
#define XSTATE_BV 0x200 |
|
3433 |
#endif |
|
3434 |
||
14626 | 3435 |
void MacroAssembler::pop_CPU_state() { |
3436 |
pop_FPU_state(); |
|
3437 |
pop_IU_state(); |
|
3438 |
} |
|
3439 |
||
3440 |
void MacroAssembler::pop_FPU_state() { |
|
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
3441 |
#ifndef _LP64 |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
3442 |
frstor(Address(rsp, 0)); |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
3443 |
#else |
34162 | 3444 |
fxrstor(Address(rsp, 0)); |
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
3445 |
#endif |
14626 | 3446 |
addptr(rsp, FPUStateSizeInWords * wordSize); |
3447 |
} |
|
3448 |
||
3449 |
void MacroAssembler::pop_IU_state() { |
|
3450 |
popa(); |
|
3451 |
LP64_ONLY(addq(rsp, 8)); |
|
3452 |
popf(); |
|
3453 |
} |
|
3454 |
||
3455 |
// Save Integer and Float state |
|
3456 |
// Warning: Stack must be 16 byte aligned (64bit) |
|
3457 |
void MacroAssembler::push_CPU_state() { |
|
3458 |
push_IU_state(); |
|
3459 |
push_FPU_state(); |
|
3460 |
} |
|
3461 |
||
3462 |
void MacroAssembler::push_FPU_state() { |
|
3463 |
subptr(rsp, FPUStateSizeInWords * wordSize); |
|
3464 |
#ifndef _LP64 |
|
3465 |
fnsave(Address(rsp, 0)); |
|
3466 |
fwait(); |
|
3467 |
#else |
|
34162 | 3468 |
fxsave(Address(rsp, 0)); |
14626 | 3469 |
#endif // LP64 |
3470 |
} |
|
3471 |
||
3472 |
void MacroAssembler::push_IU_state() { |
|
3473 |
// Push flags first because pusha kills them |
|
3474 |
pushf(); |
|
3475 |
// Make sure rsp stays 16-byte aligned |
|
3476 |
LP64_ONLY(subq(rsp, 8)); |
|
3477 |
pusha(); |
|
3478 |
} |
|
3479 |
||
40644
39e631ed7145
8161598: Kitchensink fails: assert(nm->insts_contains(original_pc)) failed: original PC must be in nmethod/CompiledMethod
dlong
parents:
39256
diff
changeset
|
3480 |
void MacroAssembler::reset_last_Java_frame(Register java_thread, bool clear_fp) { // determine java_thread register |
14626 | 3481 |
if (!java_thread->is_valid()) { |
3482 |
java_thread = rdi; |
|
3483 |
get_thread(java_thread); |
|
3484 |
} |
|
3485 |
// we must set sp to zero to clear frame |
|
3486 |
movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), NULL_WORD); |
|
3487 |
if (clear_fp) { |
|
3488 |
movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), NULL_WORD); |
|
3489 |
} |
|
3490 |
||
40644
39e631ed7145
8161598: Kitchensink fails: assert(nm->insts_contains(original_pc)) failed: original PC must be in nmethod/CompiledMethod
dlong
parents:
39256
diff
changeset
|
3491 |
// Always clear the pc because it could have been set by make_walkable() |
39e631ed7145
8161598: Kitchensink fails: assert(nm->insts_contains(original_pc)) failed: original PC must be in nmethod/CompiledMethod
dlong
parents:
39256
diff
changeset
|
3492 |
movptr(Address(java_thread, JavaThread::last_Java_pc_offset()), NULL_WORD); |
14626 | 3493 |
|
46440
61025eecb743
8178811: Minimize the AVX <-> SSE transition penalty through generation of vzeroupper instruction on x86
vdeshpande
parents:
44406
diff
changeset
|
3494 |
vzeroupper(); |
14626 | 3495 |
} |
3496 |
||
3497 |
void MacroAssembler::restore_rax(Register tmp) { |
|
3498 |
if (tmp == noreg) pop(rax); |
|
3499 |
else if (tmp != rax) mov(rax, tmp); |
|
3500 |
} |
|
3501 |
||
3502 |
void MacroAssembler::round_to(Register reg, int modulus) { |
|
3503 |
addptr(reg, modulus - 1); |
|
3504 |
andptr(reg, -modulus); |
|
3505 |
} |
|
3506 |
||
3507 |
void MacroAssembler::save_rax(Register tmp) { |
|
3508 |
if (tmp == noreg) push(rax); |
|
3509 |
else if (tmp != rax) mov(tmp, rax); |
|
3510 |
} |
|
3511 |
||
47881
0ce0ac68ace7
8189941: Implementation JEP 312: Thread-local handshake
rehn
parents:
47765
diff
changeset
|
3512 |
void MacroAssembler::safepoint_poll(Label& slow_path, Register thread_reg, Register temp_reg) { |
0ce0ac68ace7
8189941: Implementation JEP 312: Thread-local handshake
rehn
parents:
47765
diff
changeset
|
3513 |
if (SafepointMechanism::uses_thread_local_poll()) { |
49027
8dc742d9bbab
8195112: x86 (32 bit): implementation for Thread-local handshakes
mdoerr
parents:
49010
diff
changeset
|
3514 |
#ifdef _LP64 |
8dc742d9bbab
8195112: x86 (32 bit): implementation for Thread-local handshakes
mdoerr
parents:
49010
diff
changeset
|
3515 |
assert(thread_reg == r15_thread, "should be"); |
8dc742d9bbab
8195112: x86 (32 bit): implementation for Thread-local handshakes
mdoerr
parents:
49010
diff
changeset
|
3516 |
#else |
8dc742d9bbab
8195112: x86 (32 bit): implementation for Thread-local handshakes
mdoerr
parents:
49010
diff
changeset
|
3517 |
if (thread_reg == noreg) { |
8dc742d9bbab
8195112: x86 (32 bit): implementation for Thread-local handshakes
mdoerr
parents:
49010
diff
changeset
|
3518 |
thread_reg = temp_reg; |
8dc742d9bbab
8195112: x86 (32 bit): implementation for Thread-local handshakes
mdoerr
parents:
49010
diff
changeset
|
3519 |
get_thread(thread_reg); |
8dc742d9bbab
8195112: x86 (32 bit): implementation for Thread-local handshakes
mdoerr
parents:
49010
diff
changeset
|
3520 |
} |
8dc742d9bbab
8195112: x86 (32 bit): implementation for Thread-local handshakes
mdoerr
parents:
49010
diff
changeset
|
3521 |
#endif |
8dc742d9bbab
8195112: x86 (32 bit): implementation for Thread-local handshakes
mdoerr
parents:
49010
diff
changeset
|
3522 |
testb(Address(thread_reg, Thread::polling_page_offset()), SafepointMechanism::poll_bit()); |
47881
0ce0ac68ace7
8189941: Implementation JEP 312: Thread-local handshake
rehn
parents:
47765
diff
changeset
|
3523 |
jcc(Assembler::notZero, slow_path); // handshake bit set implies poll |
0ce0ac68ace7
8189941: Implementation JEP 312: Thread-local handshake
rehn
parents:
47765
diff
changeset
|
3524 |
} else { |
0ce0ac68ace7
8189941: Implementation JEP 312: Thread-local handshake
rehn
parents:
47765
diff
changeset
|
3525 |
cmp32(ExternalAddress(SafepointSynchronize::address_of_state()), |
0ce0ac68ace7
8189941: Implementation JEP 312: Thread-local handshake
rehn
parents:
47765
diff
changeset
|
3526 |
SafepointSynchronize::_not_synchronized); |
0ce0ac68ace7
8189941: Implementation JEP 312: Thread-local handshake
rehn
parents:
47765
diff
changeset
|
3527 |
jcc(Assembler::notEqual, slow_path); |
0ce0ac68ace7
8189941: Implementation JEP 312: Thread-local handshake
rehn
parents:
47765
diff
changeset
|
3528 |
} |
0ce0ac68ace7
8189941: Implementation JEP 312: Thread-local handshake
rehn
parents:
47765
diff
changeset
|
3529 |
} |
0ce0ac68ace7
8189941: Implementation JEP 312: Thread-local handshake
rehn
parents:
47765
diff
changeset
|
3530 |
|
14626 | 3531 |
// Calls to C land |
3532 |
// |
|
3533 |
// When entering C land, the rbp, & rsp of the last Java frame have to be recorded |
|
3534 |
// in the (thread-local) JavaThread object. When leaving C land, the last Java fp |
|
3535 |
// has to be reset to 0. This is required to allow proper stack traversal. |
|
3536 |
void MacroAssembler::set_last_Java_frame(Register java_thread, |
|
3537 |
Register last_java_sp, |
|
3538 |
Register last_java_fp, |
|
3539 |
address last_java_pc) { |
|
46440
61025eecb743
8178811: Minimize the AVX <-> SSE transition penalty through generation of vzeroupper instruction on x86
vdeshpande
parents:
44406
diff
changeset
|
3540 |
vzeroupper(); |
14626 | 3541 |
// determine java_thread register |
3542 |
if (!java_thread->is_valid()) { |
|
3543 |
java_thread = rdi; |
|
3544 |
get_thread(java_thread); |
|
3545 |
} |
|
3546 |
// determine last_java_sp register |
|
3547 |
if (!last_java_sp->is_valid()) { |
|
3548 |
last_java_sp = rsp; |
|
3549 |
} |
|
3550 |
||
3551 |
// last_java_fp is optional |
|
3552 |
||
3553 |
if (last_java_fp->is_valid()) { |
|
3554 |
movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), last_java_fp); |
|
3555 |
} |
|
3556 |
||
3557 |
// last_java_pc is optional |
|
3558 |
||
3559 |
if (last_java_pc != NULL) { |
|
3560 |
lea(Address(java_thread, |
|
3561 |
JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset()), |
|
3562 |
InternalAddress(last_java_pc)); |
|
3563 |
||
3564 |
} |
|
3565 |
movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), last_java_sp); |
|
3566 |
} |
|
3567 |
||
3568 |
void MacroAssembler::shlptr(Register dst, int imm8) { |
|
3569 |
LP64_ONLY(shlq(dst, imm8)) NOT_LP64(shll(dst, imm8)); |
|
3570 |
} |
|
3571 |
||
3572 |
void MacroAssembler::shrptr(Register dst, int imm8) { |
|
3573 |
LP64_ONLY(shrq(dst, imm8)) NOT_LP64(shrl(dst, imm8)); |
|
3574 |
} |
|
3575 |
||
3576 |
void MacroAssembler::sign_extend_byte(Register reg) { |
|
3577 |
if (LP64_ONLY(true ||) (VM_Version::is_P6() && reg->has_byte_register())) { |
|
3578 |
movsbl(reg, reg); // movsxb |
|
3579 |
} else { |
|
3580 |
shll(reg, 24); |
|
3581 |
sarl(reg, 24); |
|
3582 |
} |
|
3583 |
} |
|
3584 |
||
3585 |
void MacroAssembler::sign_extend_short(Register reg) { |
|
3586 |
if (LP64_ONLY(true ||) VM_Version::is_P6()) { |
|
3587 |
movswl(reg, reg); // movsxw |
|
3588 |
} else { |
|
3589 |
shll(reg, 16); |
|
3590 |
sarl(reg, 16); |
|
3591 |
} |
|
3592 |
} |
|
3593 |
||
3594 |
void MacroAssembler::testl(Register dst, AddressLiteral src) { |
|
3595 |
assert(reachable(src), "Address should be reachable"); |
|
3596 |
testl(dst, as_Address(src)); |
|
3597 |
} |
|
3598 |
||
34203 | 3599 |
void MacroAssembler::pcmpeqb(XMMRegister dst, XMMRegister src) { |
51857 | 3600 |
assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); |
3601 |
Assembler::pcmpeqb(dst, src); |
|
34203 | 3602 |
} |
3603 |
||
3604 |
void MacroAssembler::pcmpeqw(XMMRegister dst, XMMRegister src) { |
|
51857 | 3605 |
assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); |
3606 |
Assembler::pcmpeqw(dst, src); |
|
34203 | 3607 |
} |
3608 |
||
3609 |
void MacroAssembler::pcmpestri(XMMRegister dst, Address src, int imm8) { |
|
51857 | 3610 |
assert((dst->encoding() < 16),"XMM register should be 0-15"); |
3611 |
Assembler::pcmpestri(dst, src, imm8); |
|
34203 | 3612 |
} |
3613 |
||
3614 |
void MacroAssembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) { |
|
51857 | 3615 |
assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15"); |
3616 |
Assembler::pcmpestri(dst, src, imm8); |
|
34203 | 3617 |
} |
3618 |
||
3619 |
void MacroAssembler::pmovzxbw(XMMRegister dst, XMMRegister src) { |
|
51857 | 3620 |
assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); |
3621 |
Assembler::pmovzxbw(dst, src); |
|
34203 | 3622 |
} |
3623 |
||
3624 |
void MacroAssembler::pmovzxbw(XMMRegister dst, Address src) { |
|
51857 | 3625 |
assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); |
3626 |
Assembler::pmovzxbw(dst, src); |
|
34203 | 3627 |
} |
3628 |
||
3629 |
void MacroAssembler::pmovmskb(Register dst, XMMRegister src) { |
|
51857 | 3630 |
assert((src->encoding() < 16),"XMM register should be 0-15"); |
3631 |
Assembler::pmovmskb(dst, src); |
|
34203 | 3632 |
} |
3633 |
||
3634 |
void MacroAssembler::ptest(XMMRegister dst, XMMRegister src) { |
|
51857 | 3635 |
assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15"); |
3636 |
Assembler::ptest(dst, src); |
|
34203 | 3637 |
} |
3638 |
||
14626 | 3639 |
void MacroAssembler::sqrtsd(XMMRegister dst, AddressLiteral src) { |
3640 |
if (reachable(src)) { |
|
3641 |
Assembler::sqrtsd(dst, as_Address(src)); |
|
3642 |
} else { |
|
3643 |
lea(rscratch1, src); |
|
3644 |
Assembler::sqrtsd(dst, Address(rscratch1, 0)); |
|
3645 |
} |
|
3646 |
} |
|
3647 |
||
3648 |
void MacroAssembler::sqrtss(XMMRegister dst, AddressLiteral src) { |
|
3649 |
if (reachable(src)) { |
|
3650 |
Assembler::sqrtss(dst, as_Address(src)); |
|
3651 |
} else { |
|
3652 |
lea(rscratch1, src); |
|
3653 |
Assembler::sqrtss(dst, Address(rscratch1, 0)); |
|
3654 |
} |
|
3655 |
} |
|
3656 |
||
3657 |
void MacroAssembler::subsd(XMMRegister dst, AddressLiteral src) { |
|
3658 |
if (reachable(src)) { |
|
3659 |
Assembler::subsd(dst, as_Address(src)); |
|
3660 |
} else { |
|
3661 |
lea(rscratch1, src); |
|
3662 |
Assembler::subsd(dst, Address(rscratch1, 0)); |
|
3663 |
} |
|
3664 |
} |
|
3665 |
||
58421
6fc57e391539
8226721: Missing intrinsics for Math.ceil, floor, rint
neliasso
parents:
58103
diff
changeset
|
3666 |
void MacroAssembler::roundsd(XMMRegister dst, AddressLiteral src, int32_t rmode, Register scratch_reg) { |
6fc57e391539
8226721: Missing intrinsics for Math.ceil, floor, rint
neliasso
parents:
58103
diff
changeset
|
3667 |
if (reachable(src)) { |
6fc57e391539
8226721: Missing intrinsics for Math.ceil, floor, rint
neliasso
parents:
58103
diff
changeset
|
3668 |
Assembler::roundsd(dst, as_Address(src), rmode); |
6fc57e391539
8226721: Missing intrinsics for Math.ceil, floor, rint
neliasso
parents:
58103
diff
changeset
|
3669 |
} else { |
6fc57e391539
8226721: Missing intrinsics for Math.ceil, floor, rint
neliasso
parents:
58103
diff
changeset
|
3670 |
lea(scratch_reg, src); |
6fc57e391539
8226721: Missing intrinsics for Math.ceil, floor, rint
neliasso
parents:
58103
diff
changeset
|
3671 |
Assembler::roundsd(dst, Address(scratch_reg, 0), rmode); |
6fc57e391539
8226721: Missing intrinsics for Math.ceil, floor, rint
neliasso
parents:
58103
diff
changeset
|
3672 |
} |
6fc57e391539
8226721: Missing intrinsics for Math.ceil, floor, rint
neliasso
parents:
58103
diff
changeset
|
3673 |
} |
6fc57e391539
8226721: Missing intrinsics for Math.ceil, floor, rint
neliasso
parents:
58103
diff
changeset
|
3674 |
|
14626 | 3675 |
void MacroAssembler::subss(XMMRegister dst, AddressLiteral src) { |
3676 |
if (reachable(src)) { |
|
3677 |
Assembler::subss(dst, as_Address(src)); |
|
3678 |
} else { |
|
3679 |
lea(rscratch1, src); |
|
3680 |
Assembler::subss(dst, Address(rscratch1, 0)); |
|
3681 |
} |
|
3682 |
} |
|
3683 |
||
3684 |
void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src) { |
|
3685 |
if (reachable(src)) { |
|
3686 |
Assembler::ucomisd(dst, as_Address(src)); |
|
3687 |
} else { |
|
3688 |
lea(rscratch1, src); |
|
3689 |
Assembler::ucomisd(dst, Address(rscratch1, 0)); |
|
3690 |
} |
|
3691 |
} |
|
3692 |
||
3693 |
void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src) { |
|
3694 |
if (reachable(src)) { |
|
3695 |
Assembler::ucomiss(dst, as_Address(src)); |
|
3696 |
} else { |
|
3697 |
lea(rscratch1, src); |
|
3698 |
Assembler::ucomiss(dst, Address(rscratch1, 0)); |
|
3699 |
} |
|
3700 |
} |
|
3701 |
||
54750 | 3702 |
void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src, Register scratch_reg) { |
14626 | 3703 |
// Used in sign-bit flipping with aligned address. |
3704 |
assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); |
|
3705 |
if (reachable(src)) { |
|
3706 |
Assembler::xorpd(dst, as_Address(src)); |
|
3707 |
} else { |
|
54750 | 3708 |
lea(scratch_reg, src); |
3709 |
Assembler::xorpd(dst, Address(scratch_reg, 0)); |
|
14626 | 3710 |
} |
3711 |
} |
|
3712 |
||
34162 | 3713 |
void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) { |
3714 |
if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) { |
|
3715 |
Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit); |
|
3716 |
} |
|
3717 |
else { |
|
3718 |
Assembler::xorpd(dst, src); |
|
3719 |
} |
|
3720 |
} |
|
3721 |
||
3722 |
void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) { |
|
3723 |
if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) { |
|
3724 |
Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit); |
|
3725 |
} else { |
|
3726 |
Assembler::xorps(dst, src); |
|
3727 |
} |
|
3728 |
} |
|
3729 |
||
54750 | 3730 |
void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src, Register scratch_reg) { |
14626 | 3731 |
// Used in sign-bit flipping with aligned address. |
3732 |
assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); |
|
3733 |
if (reachable(src)) { |
|
3734 |
Assembler::xorps(dst, as_Address(src)); |
|
3735 |
} else { |
|
54750 | 3736 |
lea(scratch_reg, src); |
3737 |
Assembler::xorps(dst, Address(scratch_reg, 0)); |
|
14626 | 3738 |
} |
3739 |
} |
|
3740 |
||
3741 |
void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src) { |
|
3742 |
// Used in sign-bit flipping with aligned address. |
|
14834 | 3743 |
bool aligned_adr = (((intptr_t)src.target() & 15) == 0); |
3744 |
assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes"); |
|
14626 | 3745 |
if (reachable(src)) { |
3746 |
Assembler::pshufb(dst, as_Address(src)); |
|
3747 |
} else { |
|
3748 |
lea(rscratch1, src); |
|
3749 |
Assembler::pshufb(dst, Address(rscratch1, 0)); |
|
3750 |
} |
|
3751 |
} |
|
3752 |
||
3753 |
// AVX 3-operands instructions |
|
3754 |
||
3755 |
void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) { |
|
3756 |
if (reachable(src)) { |
|
3757 |
vaddsd(dst, nds, as_Address(src)); |
|
3758 |
} else { |
|
3759 |
lea(rscratch1, src); |
|
3760 |
vaddsd(dst, nds, Address(rscratch1, 0)); |
|
3761 |
} |
|
3762 |
} |
|
3763 |
||
3764 |
void MacroAssembler::vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src) { |
|
3765 |
if (reachable(src)) { |
|
3766 |
vaddss(dst, nds, as_Address(src)); |
|
3767 |
} else { |
|
3768 |
lea(rscratch1, src); |
|
3769 |
vaddss(dst, nds, Address(rscratch1, 0)); |
|
3770 |
} |
|
3771 |
} |
|
3772 |
||
34162 | 3773 |
void MacroAssembler::vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len) { |
51857 | 3774 |
assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); |
3775 |
vandps(dst, nds, negate_field, vector_len); |
|
34162 | 3776 |
} |
3777 |
||
3778 |
void MacroAssembler::vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len) { |
|
51857 | 3779 |
assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); |
3780 |
vandpd(dst, nds, negate_field, vector_len); |
|
34162 | 3781 |
} |
3782 |
||
3783 |
void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { |
|
51857 | 3784 |
assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); |
3785 |
Assembler::vpaddb(dst, nds, src, vector_len); |
|
34162 | 3786 |
} |
3787 |
||
3788 |
void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { |
|
51857 | 3789 |
assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); |
3790 |
Assembler::vpaddb(dst, nds, src, vector_len); |
|
34162 | 3791 |
} |
3792 |
||
3793 |
void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { |
|
51857 | 3794 |
assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); |
3795 |
Assembler::vpaddw(dst, nds, src, vector_len); |
|
34162 | 3796 |
} |
3797 |
||
3798 |
void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { |
|
51857 | 3799 |
assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); |
3800 |
Assembler::vpaddw(dst, nds, src, vector_len); |
|
34162 | 3801 |
} |
3802 |
||
54750 | 3803 |
void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) { |
42039 | 3804 |
if (reachable(src)) { |
3805 |
Assembler::vpand(dst, nds, as_Address(src), vector_len); |
|
3806 |
} else { |
|
54750 | 3807 |
lea(scratch_reg, src); |
3808 |
Assembler::vpand(dst, nds, Address(scratch_reg, 0), vector_len); |
|
42039 | 3809 |
} |
3810 |
} |
|
3811 |
||
51857 | 3812 |
void MacroAssembler::vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) { |
3813 |
assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); |
|
3814 |
Assembler::vpbroadcastw(dst, src, vector_len); |
|
34203 | 3815 |
} |
3816 |
||
3817 |
void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { |
|
51857 | 3818 |
assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); |
3819 |
Assembler::vpcmpeqb(dst, nds, src, vector_len); |
|
34203 | 3820 |
} |
3821 |
||
3822 |
void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { |
|
51857 | 3823 |
assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); |
3824 |
Assembler::vpcmpeqw(dst, nds, src, vector_len); |
|
34203 | 3825 |
} |
3826 |
||
3827 |
void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) { |
|
51857 | 3828 |
assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); |
3829 |
Assembler::vpmovzxbw(dst, src, vector_len); |
|
34203 | 3830 |
} |
3831 |
||
3832 |
void MacroAssembler::vpmovmskb(Register dst, XMMRegister src) { |
|
51857 | 3833 |
assert((src->encoding() < 16),"XMM register should be 0-15"); |
3834 |
Assembler::vpmovmskb(dst, src); |
|
34203 | 3835 |
} |
3836 |
||
3837 |
void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { |
|
51857 | 3838 |
assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); |
3839 |
Assembler::vpmullw(dst, nds, src, vector_len); |
|
34203 | 3840 |
} |
3841 |
||
3842 |
void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { |
|
51857 | 3843 |
assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); |
3844 |
Assembler::vpmullw(dst, nds, src, vector_len); |
|
34203 | 3845 |
} |
3846 |
||
34162 | 3847 |
void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { |
51857 | 3848 |
assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); |
3849 |
Assembler::vpsubb(dst, nds, src, vector_len); |
|
34162 | 3850 |
} |
3851 |
||
3852 |
void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { |
|
51857 | 3853 |
assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); |
3854 |
Assembler::vpsubb(dst, nds, src, vector_len); |
|
34162 | 3855 |
} |
3856 |
||
3857 |
void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { |
|
51857 | 3858 |
assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); |
3859 |
Assembler::vpsubw(dst, nds, src, vector_len); |
|
34162 | 3860 |
} |
3861 |
||
3862 |
void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { |
|
51857 | 3863 |
assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); |
3864 |
Assembler::vpsubw(dst, nds, src, vector_len); |
|
34162 | 3865 |
} |
3866 |
||
3867 |
void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { |
|
51857 | 3868 |
assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); |
3869 |
Assembler::vpsraw(dst, nds, shift, vector_len); |
|
34162 | 3870 |
} |
3871 |
||
3872 |
void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { |
|
51857 | 3873 |
assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); |
3874 |
Assembler::vpsraw(dst, nds, shift, vector_len); |
|
34162 | 3875 |
} |
3876 |
||
54750 | 3877 |
void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { |
3878 |
assert(UseAVX > 2,""); |
|
3879 |
if (!VM_Version::supports_avx512vl() && vector_len < 2) { |
|
3880 |
vector_len = 2; |
|
3881 |
} |
|
3882 |
Assembler::evpsraq(dst, nds, shift, vector_len); |
|
3883 |
} |
|
3884 |
||
3885 |
void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { |
|
3886 |
assert(UseAVX > 2,""); |
|
3887 |
if (!VM_Version::supports_avx512vl() && vector_len < 2) { |
|
3888 |
vector_len = 2; |
|
3889 |
} |
|
3890 |
Assembler::evpsraq(dst, nds, shift, vector_len); |
|
3891 |
} |
|
3892 |
||
34162 | 3893 |
void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { |
51857 | 3894 |
assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); |
3895 |
Assembler::vpsrlw(dst, nds, shift, vector_len); |
|
34162 | 3896 |
} |
3897 |
||
3898 |
void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { |
|
51857 | 3899 |
assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); |
3900 |
Assembler::vpsrlw(dst, nds, shift, vector_len); |
|
34162 | 3901 |
} |
3902 |
||
3903 |
void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { |
|
51857 | 3904 |
assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); |
3905 |
Assembler::vpsllw(dst, nds, shift, vector_len); |
|
34162 | 3906 |
} |
3907 |
||
3908 |
void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { |
|
51857 | 3909 |
assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); |
3910 |
Assembler::vpsllw(dst, nds, shift, vector_len); |
|
34162 | 3911 |
} |
3912 |
||
34203 | 3913 |
void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) { |
51857 | 3914 |
assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15"); |
3915 |
Assembler::vptest(dst, src); |
|
3916 |
} |
|
3917 |
||
34162 | 3918 |
void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) { |
51857 | 3919 |
assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); |
3920 |
Assembler::punpcklbw(dst, src); |
|
34162 | 3921 |
} |
3922 |
||
45236
1b8879e6d9c2
8178800: compiler/c2/PolynomialRoot.java fails on Xeon Phi linux host with UseAVX=3
mcberg
parents:
44406
diff
changeset
|
3923 |
void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) { |
51857 | 3924 |
assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); |
3925 |
Assembler::pshufd(dst, src, mode); |
|
3926 |
} |
|
3927 |
||
34162 | 3928 |
void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) { |
51857 | 3929 |
assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); |
3930 |
Assembler::pshuflw(dst, src, mode); |
|
34162 | 3931 |
} |
3932 |
||
54750 | 3933 |
void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) { |
14626 | 3934 |
if (reachable(src)) { |
30624 | 3935 |
vandpd(dst, nds, as_Address(src), vector_len); |
14626 | 3936 |
} else { |
54750 | 3937 |
lea(scratch_reg, src); |
3938 |
vandpd(dst, nds, Address(scratch_reg, 0), vector_len); |
|
3939 |
} |
|
3940 |
} |
|
3941 |
||
3942 |
void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) { |
|
14626 | 3943 |
if (reachable(src)) { |
30624 | 3944 |
vandps(dst, nds, as_Address(src), vector_len); |
14626 | 3945 |
} else { |
54750 | 3946 |
lea(scratch_reg, src); |
3947 |
vandps(dst, nds, Address(scratch_reg, 0), vector_len); |
|
14626 | 3948 |
} |
3949 |
} |
|
3950 |
||
3951 |
void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) { |
|
3952 |
if (reachable(src)) { |
|
3953 |
vdivsd(dst, nds, as_Address(src)); |
|
3954 |
} else { |
|
3955 |
lea(rscratch1, src); |
|
3956 |
vdivsd(dst, nds, Address(rscratch1, 0)); |
|
3957 |
} |
|
3958 |
} |
|
3959 |
||
3960 |
void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src) { |
|
3961 |
if (reachable(src)) { |
|
3962 |
vdivss(dst, nds, as_Address(src)); |
|
3963 |
} else { |
|
3964 |
lea(rscratch1, src); |
|
3965 |
vdivss(dst, nds, Address(rscratch1, 0)); |
|
3966 |
} |
|
3967 |
} |
|
3968 |
||
3969 |
void MacroAssembler::vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) { |
|
3970 |
if (reachable(src)) { |
|
3971 |
vmulsd(dst, nds, as_Address(src)); |
|
3972 |
} else { |
|
3973 |
lea(rscratch1, src); |
|
3974 |
vmulsd(dst, nds, Address(rscratch1, 0)); |
|
3975 |
} |
|
3976 |
} |
|
3977 |
||
3978 |
void MacroAssembler::vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src) { |
|
3979 |
if (reachable(src)) { |
|
3980 |
vmulss(dst, nds, as_Address(src)); |
|
3981 |
} else { |
|
3982 |
lea(rscratch1, src); |
|
3983 |
vmulss(dst, nds, Address(rscratch1, 0)); |
|
3984 |
} |
|
3985 |
} |
|
3986 |
||
3987 |
void MacroAssembler::vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) { |
|
3988 |
if (reachable(src)) { |
|
3989 |
vsubsd(dst, nds, as_Address(src)); |
|
3990 |
} else { |
|
3991 |
lea(rscratch1, src); |
|
3992 |
vsubsd(dst, nds, Address(rscratch1, 0)); |
|
3993 |
} |
|
3994 |
} |
|
3995 |
||
3996 |
void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src) { |
|
3997 |
if (reachable(src)) { |
|
3998 |
vsubss(dst, nds, as_Address(src)); |
|
3999 |
} else { |
|
4000 |
lea(rscratch1, src); |
|
4001 |
vsubss(dst, nds, Address(rscratch1, 0)); |
|
4002 |
} |
|
4003 |
} |
|
4004 |
||
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
4005 |
void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src) { |
51857 | 4006 |
assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); |
4007 |
vxorps(dst, nds, src, Assembler::AVX_128bit); |
|
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
4008 |
} |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
4009 |
|
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
4010 |
void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src) { |
51857 | 4011 |
assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); |
4012 |
vxorpd(dst, nds, src, Assembler::AVX_128bit); |
|
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
4013 |
} |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
4014 |
|
54750 | 4015 |
void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) { |
14626 | 4016 |
if (reachable(src)) { |
30624 | 4017 |
vxorpd(dst, nds, as_Address(src), vector_len); |
14626 | 4018 |
} else { |
54750 | 4019 |
lea(scratch_reg, src); |
4020 |
vxorpd(dst, nds, Address(scratch_reg, 0), vector_len); |
|
4021 |
} |
|
4022 |
} |
|
4023 |
||
4024 |
void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) { |
|
14626 | 4025 |
if (reachable(src)) { |
30624 | 4026 |
vxorps(dst, nds, as_Address(src), vector_len); |
14626 | 4027 |
} else { |
54750 | 4028 |
lea(scratch_reg, src); |
4029 |
vxorps(dst, nds, Address(scratch_reg, 0), vector_len); |
|
4030 |
} |
|
4031 |
} |
|
4032 |
||
4033 |
void MacroAssembler::vpxor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) { |
|
4034 |
if (UseAVX > 1 || (vector_len < 1)) { |
|
4035 |
if (reachable(src)) { |
|
4036 |
Assembler::vpxor(dst, nds, as_Address(src), vector_len); |
|
4037 |
} else { |
|
4038 |
lea(scratch_reg, src); |
|
4039 |
Assembler::vpxor(dst, nds, Address(scratch_reg, 0), vector_len); |
|
4040 |
} |
|
4041 |
} |
|
4042 |
else { |
|
4043 |
MacroAssembler::vxorpd(dst, nds, src, vector_len, scratch_reg); |
|
4044 |
} |
|
4045 |
} |
|
4046 |
||
4047 |
//------------------------------------------------------------------------------------------- |
|
4048 |
#ifdef COMPILER2 |
|
4049 |
// Generic instructions support for use in .ad files C2 code generation |
|
4050 |
||
4051 |
void MacroAssembler::vabsnegd(int opcode, XMMRegister dst, Register scr) { |
|
4052 |
if (opcode == Op_AbsVD) { |
|
4053 |
andpd(dst, ExternalAddress(StubRoutines::x86::vector_double_sign_mask()), scr); |
|
4054 |
} else { |
|
4055 |
assert((opcode == Op_NegVD),"opcode should be Op_NegD"); |
|
4056 |
xorpd(dst, ExternalAddress(StubRoutines::x86::vector_double_sign_flip()), scr); |
|
4057 |
} |
|
4058 |
} |
|
4059 |
||
4060 |
void MacroAssembler::vabsnegd(int opcode, XMMRegister dst, XMMRegister src, int vector_len, Register scr) { |
|
4061 |
if (opcode == Op_AbsVD) { |
|
4062 |
vandpd(dst, src, ExternalAddress(StubRoutines::x86::vector_double_sign_mask()), vector_len, scr); |
|
4063 |
} else { |
|
4064 |
assert((opcode == Op_NegVD),"opcode should be Op_NegD"); |
|
4065 |
vxorpd(dst, src, ExternalAddress(StubRoutines::x86::vector_double_sign_flip()), vector_len, scr); |
|
4066 |
} |
|
4067 |
} |
|
4068 |
||
4069 |
void MacroAssembler::vabsnegf(int opcode, XMMRegister dst, Register scr) { |
|
4070 |
if (opcode == Op_AbsVF) { |
|
4071 |
andps(dst, ExternalAddress(StubRoutines::x86::vector_float_sign_mask()), scr); |
|
4072 |
} else { |
|
4073 |
assert((opcode == Op_NegVF),"opcode should be Op_NegF"); |
|
4074 |
xorps(dst, ExternalAddress(StubRoutines::x86::vector_float_sign_flip()), scr); |
|
4075 |
} |
|
4076 |
} |
|
4077 |
||
4078 |
void MacroAssembler::vabsnegf(int opcode, XMMRegister dst, XMMRegister src, int vector_len, Register scr) { |
|
4079 |
if (opcode == Op_AbsVF) { |
|
4080 |
vandps(dst, src, ExternalAddress(StubRoutines::x86::vector_float_sign_mask()), vector_len, scr); |
|
4081 |
} else { |
|
4082 |
assert((opcode == Op_NegVF),"opcode should be Op_NegF"); |
|
4083 |
vxorps(dst, src, ExternalAddress(StubRoutines::x86::vector_float_sign_flip()), vector_len, scr); |
|
4084 |
} |
|
4085 |
} |
|
4086 |
||
4087 |
void MacroAssembler::vextendbw(bool sign, XMMRegister dst, XMMRegister src) { |
|
4088 |
if (sign) { |
|
4089 |
pmovsxbw(dst, src); |
|
4090 |
} else { |
|
4091 |
pmovzxbw(dst, src); |
|
4092 |
} |
|
4093 |
} |
|
4094 |
||
4095 |
void MacroAssembler::vextendbw(bool sign, XMMRegister dst, XMMRegister src, int vector_len) { |
|
4096 |
if (sign) { |
|
4097 |
vpmovsxbw(dst, src, vector_len); |
|
4098 |
} else { |
|
4099 |
vpmovzxbw(dst, src, vector_len); |
|
4100 |
} |
|
4101 |
} |
|
4102 |
||
4103 |
void MacroAssembler::vshiftd(int opcode, XMMRegister dst, XMMRegister src) { |
|
4104 |
if (opcode == Op_RShiftVI) { |
|
4105 |
psrad(dst, src); |
|
4106 |
} else if (opcode == Op_LShiftVI) { |
|
4107 |
pslld(dst, src); |
|
4108 |
} else { |
|
4109 |
assert((opcode == Op_URShiftVI),"opcode should be Op_URShiftVI"); |
|
4110 |
psrld(dst, src); |
|
4111 |
} |
|
4112 |
} |
|
4113 |
||
4114 |
void MacroAssembler::vshiftd(int opcode, XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { |
|
4115 |
if (opcode == Op_RShiftVI) { |
|
4116 |
vpsrad(dst, nds, src, vector_len); |
|
4117 |
} else if (opcode == Op_LShiftVI) { |
|
4118 |
vpslld(dst, nds, src, vector_len); |
|
4119 |
} else { |
|
4120 |
assert((opcode == Op_URShiftVI),"opcode should be Op_URShiftVI"); |
|
4121 |
vpsrld(dst, nds, src, vector_len); |
|
4122 |
} |
|
4123 |
} |
|
4124 |
||
4125 |
void MacroAssembler::vshiftw(int opcode, XMMRegister dst, XMMRegister src) { |
|
4126 |
if ((opcode == Op_RShiftVS) || (opcode == Op_RShiftVB)) { |
|
4127 |
psraw(dst, src); |
|
4128 |
} else if ((opcode == Op_LShiftVS) || (opcode == Op_LShiftVB)) { |
|
4129 |
psllw(dst, src); |
|
4130 |
} else { |
|
4131 |
assert(((opcode == Op_URShiftVS) || (opcode == Op_URShiftVB)),"opcode should be one of Op_URShiftVS or Op_URShiftVB"); |
|
4132 |
psrlw(dst, src); |
|
4133 |
} |
|
4134 |
} |
|
4135 |
||
4136 |
void MacroAssembler::vshiftw(int opcode, XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { |
|
4137 |
if ((opcode == Op_RShiftVS) || (opcode == Op_RShiftVB)) { |
|
4138 |
vpsraw(dst, nds, src, vector_len); |
|
4139 |
} else if ((opcode == Op_LShiftVS) || (opcode == Op_LShiftVB)) { |
|
4140 |
vpsllw(dst, nds, src, vector_len); |
|
4141 |
} else { |
|
4142 |
assert(((opcode == Op_URShiftVS) || (opcode == Op_URShiftVB)),"opcode should be one of Op_URShiftVS or Op_URShiftVB"); |
|
4143 |
vpsrlw(dst, nds, src, vector_len); |
|
4144 |
} |
|
4145 |
} |
|
4146 |
||
4147 |
void MacroAssembler::vshiftq(int opcode, XMMRegister dst, XMMRegister src) { |
|
4148 |
if (opcode == Op_RShiftVL) { |
|
4149 |
psrlq(dst, src); // using srl to implement sra on pre-avs512 systems |
|
4150 |
} else if (opcode == Op_LShiftVL) { |
|
4151 |
psllq(dst, src); |
|
4152 |
} else { |
|
4153 |
assert((opcode == Op_URShiftVL),"opcode should be Op_URShiftVL"); |
|
4154 |
psrlq(dst, src); |
|
4155 |
} |
|
4156 |
} |
|
4157 |
||
4158 |
void MacroAssembler::vshiftq(int opcode, XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { |
|
4159 |
if (opcode == Op_RShiftVL) { |
|
4160 |
evpsraq(dst, nds, src, vector_len); |
|
4161 |
} else if (opcode == Op_LShiftVL) { |
|
4162 |
vpsllq(dst, nds, src, vector_len); |
|
4163 |
} else { |
|
4164 |
assert((opcode == Op_URShiftVL),"opcode should be Op_URShiftVL"); |
|
4165 |
vpsrlq(dst, nds, src, vector_len); |
|
4166 |
} |
|
4167 |
} |
|
4168 |
#endif |
|
4169 |
//------------------------------------------------------------------------------------------- |
|
14626 | 4170 |
|
49748 | 4171 |
void MacroAssembler::clear_jweak_tag(Register possibly_jweak) { |
4172 |
const int32_t inverted_jweak_mask = ~static_cast<int32_t>(JNIHandles::weak_tag_mask); |
|
4173 |
STATIC_ASSERT(inverted_jweak_mask == -2); // otherwise check this code |
|
4174 |
// The inverted mask is sign-extended |
|
4175 |
andptr(possibly_jweak, inverted_jweak_mask); |
|
4176 |
} |
|
14626 | 4177 |
|
44406
a46a6c4d1dd9
8176100: [REDO][REDO] G1 Needs pre barrier on dereference of weak JNI handles
mgerdin
parents:
44093
diff
changeset
|
4178 |
void MacroAssembler::resolve_jobject(Register value, |
a46a6c4d1dd9
8176100: [REDO][REDO] G1 Needs pre barrier on dereference of weak JNI handles
mgerdin
parents:
44093
diff
changeset
|
4179 |
Register thread, |
a46a6c4d1dd9
8176100: [REDO][REDO] G1 Needs pre barrier on dereference of weak JNI handles
mgerdin
parents:
44093
diff
changeset
|
4180 |
Register tmp) { |
a46a6c4d1dd9
8176100: [REDO][REDO] G1 Needs pre barrier on dereference of weak JNI handles
mgerdin
parents:
44093
diff
changeset
|
4181 |
assert_different_registers(value, thread, tmp); |
a46a6c4d1dd9
8176100: [REDO][REDO] G1 Needs pre barrier on dereference of weak JNI handles
mgerdin
parents:
44093
diff
changeset
|
4182 |
Label done, not_weak; |
a46a6c4d1dd9
8176100: [REDO][REDO] G1 Needs pre barrier on dereference of weak JNI handles
mgerdin
parents:
44093
diff
changeset
|
4183 |
testptr(value, value); |
a46a6c4d1dd9
8176100: [REDO][REDO] G1 Needs pre barrier on dereference of weak JNI handles
mgerdin
parents:
44093
diff
changeset
|
4184 |
jcc(Assembler::zero, done); // Use NULL as-is. |
a46a6c4d1dd9
8176100: [REDO][REDO] G1 Needs pre barrier on dereference of weak JNI handles
mgerdin
parents:
44093
diff
changeset
|
4185 |
testptr(value, JNIHandles::weak_tag_mask); // Test for jweak tag. |
a46a6c4d1dd9
8176100: [REDO][REDO] G1 Needs pre barrier on dereference of weak JNI handles
mgerdin
parents:
44093
diff
changeset
|
4186 |
jcc(Assembler::zero, not_weak); |
a46a6c4d1dd9
8176100: [REDO][REDO] G1 Needs pre barrier on dereference of weak JNI handles
mgerdin
parents:
44093
diff
changeset
|
4187 |
// Resolve jweak. |
50599
ecc2af326b5f
8204939: Change Access nomenclature: root to native
kbarrett
parents:
50536
diff
changeset
|
4188 |
access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, |
49748 | 4189 |
value, Address(value, -JNIHandles::weak_tag_value), tmp, thread); |
44406
a46a6c4d1dd9
8176100: [REDO][REDO] G1 Needs pre barrier on dereference of weak JNI handles
mgerdin
parents:
44093
diff
changeset
|
4190 |
verify_oop(value); |
a46a6c4d1dd9
8176100: [REDO][REDO] G1 Needs pre barrier on dereference of weak JNI handles
mgerdin
parents:
44093
diff
changeset
|
4191 |
jmp(done); |
a46a6c4d1dd9
8176100: [REDO][REDO] G1 Needs pre barrier on dereference of weak JNI handles
mgerdin
parents:
44093
diff
changeset
|
4192 |
bind(not_weak); |
a46a6c4d1dd9
8176100: [REDO][REDO] G1 Needs pre barrier on dereference of weak JNI handles
mgerdin
parents:
44093
diff
changeset
|
4193 |
// Resolve (untagged) jobject. |
50803
45c1fde86050
8205559: Remove IN_CONCURRENT_ROOT Access decorator
kbarrett
parents:
50728
diff
changeset
|
4194 |
access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, 0), tmp, thread); |
44406
a46a6c4d1dd9
8176100: [REDO][REDO] G1 Needs pre barrier on dereference of weak JNI handles
mgerdin
parents:
44093
diff
changeset
|
4195 |
verify_oop(value); |
a46a6c4d1dd9
8176100: [REDO][REDO] G1 Needs pre barrier on dereference of weak JNI handles
mgerdin
parents:
44093
diff
changeset
|
4196 |
bind(done); |
a46a6c4d1dd9
8176100: [REDO][REDO] G1 Needs pre barrier on dereference of weak JNI handles
mgerdin
parents:
44093
diff
changeset
|
4197 |
} |
a46a6c4d1dd9
8176100: [REDO][REDO] G1 Needs pre barrier on dereference of weak JNI handles
mgerdin
parents:
44093
diff
changeset
|
4198 |
|
14626 | 4199 |
void MacroAssembler::subptr(Register dst, int32_t imm32) { |
4200 |
LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32)); |
|
4201 |
} |
|
4202 |
||
4203 |
// Force generation of a 4 byte immediate value even if it fits into 8bit |
|
4204 |
void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) { |
|
4205 |
LP64_ONLY(subq_imm32(dst, imm32)) NOT_LP64(subl_imm32(dst, imm32)); |
|
4206 |
} |
|
4207 |
||
4208 |
void MacroAssembler::subptr(Register dst, Register src) { |
|
4209 |
LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src)); |
|
4210 |
} |
|
4211 |
||
4212 |
// C++ bool manipulation |
|
4213 |
void MacroAssembler::testbool(Register dst) { |
|
4214 |
if(sizeof(bool) == 1) |
|
4215 |
testb(dst, 0xff); |
|
4216 |
else if(sizeof(bool) == 2) { |
|
4217 |
// testw implementation needed for two byte bools |
|
4218 |
ShouldNotReachHere(); |
|
4219 |
} else if(sizeof(bool) == 4) |
|
4220 |
testl(dst, dst); |
|
4221 |
else |
|
4222 |
// unsupported |
|
4223 |
ShouldNotReachHere(); |
|
4224 |
} |
|
4225 |
||
4226 |
void MacroAssembler::testptr(Register dst, Register src) { |
|
4227 |
LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src)); |
|
4228 |
} |
|
4229 |
||
4230 |
// Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes. |
|
50693 | 4231 |
void MacroAssembler::tlab_allocate(Register thread, Register obj, |
14626 | 4232 |
Register var_size_in_bytes, |
4233 |
int con_size_in_bytes, |
|
4234 |
Register t1, |
|
4235 |
Register t2, |
|
4236 |
Label& slow_case) { |
|
50693 | 4237 |
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); |
4238 |
bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case); |
|
4239 |
} |
|
4240 |
||
4241 |
// Defines obj, preserves var_size_in_bytes |
|
4242 |
void MacroAssembler::eden_allocate(Register thread, Register obj, |
|
4243 |
Register var_size_in_bytes, |
|
4244 |
int con_size_in_bytes, |
|
4245 |
Register t1, |
|
4246 |
Label& slow_case) { |
|
4247 |
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); |
|
4248 |
bs->eden_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case); |
|
14626 | 4249 |
} |
4250 |
||
35548
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4251 |
// Preserves the contents of address, destroys the contents length_in_bytes and temp. |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4252 |
void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) { |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4253 |
assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different"); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4254 |
assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord"); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4255 |
Label done; |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4256 |
|
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4257 |
testptr(length_in_bytes, length_in_bytes); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4258 |
jcc(Assembler::zero, done); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4259 |
|
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4260 |
// initialize topmost word, divide index by 2, check if odd and test if zero |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4261 |
// note: for the remaining code to work, index must be a multiple of BytesPerWord |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4262 |
#ifdef ASSERT |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4263 |
{ |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4264 |
Label L; |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4265 |
testptr(length_in_bytes, BytesPerWord - 1); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4266 |
jcc(Assembler::zero, L); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4267 |
stop("length must be a multiple of BytesPerWord"); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4268 |
bind(L); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4269 |
} |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4270 |
#endif |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4271 |
Register index = length_in_bytes; |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4272 |
xorptr(temp, temp); // use _zero reg to clear memory (shorter code) |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4273 |
if (UseIncDec) { |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4274 |
shrptr(index, 3); // divide by 8/16 and set carry flag if bit 2 was set |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4275 |
} else { |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4276 |
shrptr(index, 2); // use 2 instructions to avoid partial flag stall |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4277 |
shrptr(index, 1); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4278 |
} |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4279 |
#ifndef _LP64 |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4280 |
// index could have not been a multiple of 8 (i.e., bit 2 was set) |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4281 |
{ |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4282 |
Label even; |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4283 |
// note: if index was a multiple of 8, then it cannot |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4284 |
// be 0 now otherwise it must have been 0 before |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4285 |
// => if it is even, we don't need to check for 0 again |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4286 |
jcc(Assembler::carryClear, even); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4287 |
// clear topmost word (no jump would be needed if conditional assignment worked here) |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4288 |
movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4289 |
// index could be 0 now, must check again |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4290 |
jcc(Assembler::zero, done); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4291 |
bind(even); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4292 |
} |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4293 |
#endif // !_LP64 |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4294 |
// initialize remaining object fields: index is a multiple of 2 now |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4295 |
{ |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4296 |
Label loop; |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4297 |
bind(loop); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4298 |
movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4299 |
NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);) |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4300 |
decrement(index); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4301 |
jcc(Assembler::notZero, loop); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4302 |
} |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4303 |
|
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4304 |
bind(done); |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4305 |
} |
8d3afe96ffea
8086053: Address inconsistencies regarding ZeroTLAB
zmajo
parents:
35546
diff
changeset
|
4306 |
|
14626 | 4307 |
// Look up the method for a megamorphic invokeinterface call. |
4308 |
// The target method is determined by <intf_klass, itable_index>. |
|
4309 |
// The receiver klass is in recv_klass. |
|
4310 |
// On success, the result will be in method_result, and execution falls through. |
|
4311 |
// On failure, execution transfers to the given label. |
|
4312 |
void MacroAssembler::lookup_interface_method(Register recv_klass, |
|
4313 |
Register intf_klass, |
|
4314 |
RegisterOrConstant itable_index, |
|
4315 |
Register method_result, |
|
4316 |
Register scan_temp, |
|
48557 | 4317 |
Label& L_no_such_interface, |
4318 |
bool return_method) { |
|
4319 |
assert_different_registers(recv_klass, intf_klass, scan_temp); |
|
4320 |
assert_different_registers(method_result, intf_klass, scan_temp); |
|
4321 |
assert(recv_klass != method_result || !return_method, |
|
4322 |
"recv_klass can be destroyed when method isn't needed"); |
|
4323 |
||
14626 | 4324 |
assert(itable_index.is_constant() || itable_index.as_register() == method_result, |
4325 |
"caller must use same register for non-constant itable index as for method"); |
|
4326 |
||
4327 |
// Compute start of first itableOffsetEntry (which is at the end of the vtable) |
|
35899 | 4328 |
int vtable_base = in_bytes(Klass::vtable_start_offset()); |
14626 | 4329 |
int itentry_off = itableMethodEntry::method_offset_in_bytes(); |
4330 |
int scan_step = itableOffsetEntry::size() * wordSize; |
|
35871
607bf949dfb3
8147461: Use byte offsets for vtable start and vtable length offsets
mgerdin
parents:
35847
diff
changeset
|
4331 |
int vte_size = vtableEntry::size_in_bytes(); |
14626 | 4332 |
Address::ScaleFactor times_vte_scale = Address::times_ptr; |
4333 |
assert(vte_size == wordSize, "else adjust times_vte_scale"); |
|
4334 |
||
35899 | 4335 |
movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset())); |
14626 | 4336 |
|
4337 |
// %%% Could store the aligned, prescaled offset in the klassoop. |
|
4338 |
lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base)); |
|
4339 |
||
48557 | 4340 |
if (return_method) { |
4341 |
// Adjust recv_klass by scaled itable_index, so we can free itable_index. |
|
4342 |
assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below"); |
|
4343 |
lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off)); |
|
4344 |
} |
|
14626 | 4345 |
|
4346 |
// for (scan = klass->itable(); scan->interface() != NULL; scan += scan_step) { |
|
4347 |
// if (scan->interface() == intf) { |
|
4348 |
// result = (klass + scan->offset() + itable_index); |
|
4349 |
// } |
|
4350 |
// } |
|
4351 |
Label search, found_method; |
|
4352 |
||
4353 |
for (int peel = 1; peel >= 0; peel--) { |
|
4354 |
movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset_in_bytes())); |
|
4355 |
cmpptr(intf_klass, method_result); |
|
4356 |
||
4357 |
if (peel) { |
|
4358 |
jccb(Assembler::equal, found_method); |
|
4359 |
} else { |
|
4360 |
jccb(Assembler::notEqual, search); |
|
4361 |
// (invert the test to fall through to found_method...) |
|
4362 |
} |
|
4363 |
||
4364 |
if (!peel) break; |
|
4365 |
||
4366 |
bind(search); |
|
4367 |
||
4368 |
// Check that the previous entry is non-null. A null entry means that |
|
4369 |
// the receiver class doesn't implement the interface, and wasn't the |
|
4370 |
// same as when the caller was compiled. |
|
4371 |
testptr(method_result, method_result); |
|
4372 |
jcc(Assembler::zero, L_no_such_interface); |
|
4373 |
addptr(scan_temp, scan_step); |
|
4374 |
} |
|
4375 |
||
4376 |
bind(found_method); |
|
4377 |
||
48557 | 4378 |
if (return_method) { |
4379 |
// Got a hit. |
|
4380 |
movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset_in_bytes())); |
|
4381 |
movptr(method_result, Address(recv_klass, scan_temp, Address::times_1)); |
|
4382 |
} |
|
14626 | 4383 |
} |
4384 |
||
4385 |
||
4386 |
// virtual method calling |
|
4387 |
void MacroAssembler::lookup_virtual_method(Register recv_klass, |
|
4388 |
RegisterOrConstant vtable_index, |
|
4389 |
Register method_result) { |
|
35899 | 4390 |
const int base = in_bytes(Klass::vtable_start_offset()); |
14626 | 4391 |
assert(vtableEntry::size() * wordSize == wordSize, "else adjust the scaling in the code below"); |
4392 |
Address vtable_entry_addr(recv_klass, |
|
4393 |
vtable_index, Address::times_ptr, |
|
4394 |
base + vtableEntry::method_offset_in_bytes()); |
|
4395 |
movptr(method_result, vtable_entry_addr); |
|
4396 |
} |
|
4397 |
||
4398 |
||
4399 |
void MacroAssembler::check_klass_subtype(Register sub_klass, |
|
4400 |
Register super_klass, |
|
4401 |
Register temp_reg, |
|
4402 |
Label& L_success) { |
|
4403 |
Label L_failure; |
|
4404 |
check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg, &L_success, &L_failure, NULL); |
|
4405 |
check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, NULL); |
|
4406 |
bind(L_failure); |
|
4407 |
} |
|
4408 |
||
4409 |
||
4410 |
void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass, |
|
4411 |
Register super_klass, |
|
4412 |
Register temp_reg, |
|
4413 |
Label* L_success, |
|
4414 |
Label* L_failure, |
|
4415 |
Label* L_slow_path, |
|
4416 |
RegisterOrConstant super_check_offset) { |
|
4417 |
assert_different_registers(sub_klass, super_klass, temp_reg); |
|
4418 |
bool must_load_sco = (super_check_offset.constant_or_zero() == -1); |
|
4419 |
if (super_check_offset.is_register()) { |
|
4420 |
assert_different_registers(sub_klass, super_klass, |
|
4421 |
super_check_offset.as_register()); |
|
4422 |
} else if (must_load_sco) { |
|
4423 |
assert(temp_reg != noreg, "supply either a temp or a register offset"); |
|
4424 |
} |
|
4425 |
||
4426 |
Label L_fallthrough; |
|
4427 |
int label_nulls = 0; |
|
4428 |
if (L_success == NULL) { L_success = &L_fallthrough; label_nulls++; } |
|
4429 |
if (L_failure == NULL) { L_failure = &L_fallthrough; label_nulls++; } |
|
4430 |
if (L_slow_path == NULL) { L_slow_path = &L_fallthrough; label_nulls++; } |
|
4431 |
assert(label_nulls <= 1, "at most one NULL in the batch"); |
|
4432 |
||
4433 |
int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); |
|
4434 |
int sco_offset = in_bytes(Klass::super_check_offset_offset()); |
|
4435 |
Address super_check_offset_addr(super_klass, sco_offset); |
|
4436 |
||
4437 |
// Hacked jcc, which "knows" that L_fallthrough, at least, is in |
|
4438 |
// range of a jccb. If this routine grows larger, reconsider at |
|
4439 |
// least some of these. |
|
4440 |
#define local_jcc(assembler_cond, label) \ |
|
4441 |
if (&(label) == &L_fallthrough) jccb(assembler_cond, label); \ |
|
4442 |
else jcc( assembler_cond, label) /*omit semi*/ |
|
4443 |
||
4444 |
// Hacked jmp, which may only be used just before L_fallthrough. |
|
4445 |
#define final_jmp(label) \ |
|
4446 |
if (&(label) == &L_fallthrough) { /*do nothing*/ } \ |
|
4447 |
else jmp(label) /*omit semi*/ |
|
4448 |
||
4449 |
// If the pointers are equal, we are done (e.g., String[] elements). |
|
4450 |
// This self-check enables sharing of secondary supertype arrays among |
|
4451 |
// non-primary types such as array-of-interface. Otherwise, each such |
|
4452 |
// type would need its own customized SSA. |
|
4453 |
// We move this check to the front of the fast path because many |
|
4454 |
// type checks are in fact trivially successful in this manner, |
|
4455 |
// so we get a nicely predicted branch right at the start of the check. |
|
4456 |
cmpptr(sub_klass, super_klass); |
|
4457 |
local_jcc(Assembler::equal, *L_success); |
|
4458 |
||
4459 |
// Check the supertype display: |
|
4460 |
if (must_load_sco) { |
|
4461 |
// Positive movl does right thing on LP64. |
|
4462 |
movl(temp_reg, super_check_offset_addr); |
|
4463 |
super_check_offset = RegisterOrConstant(temp_reg); |
|
4464 |
} |
|
4465 |
Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0); |
|
4466 |
cmpptr(super_klass, super_check_addr); // load displayed supertype |
|
4467 |
||
4468 |
// This check has worked decisively for primary supers. |
|
4469 |
// Secondary supers are sought in the super_cache ('super_cache_addr'). |
|
4470 |
// (Secondary supers are interfaces and very deeply nested subtypes.) |
|
4471 |
// This works in the same check above because of a tricky aliasing |
|
4472 |
// between the super_cache and the primary super display elements. |
|
4473 |
// (The 'super_check_addr' can address either, as the case requires.) |
|
4474 |
// Note that the cache is updated below if it does not help us find |
|
4475 |
// what we need immediately. |
|
4476 |
// So if it was a primary super, we can just fail immediately. |
|
4477 |
// Otherwise, it's the slow path for us (no success at this point). |
|
4478 |
||
4479 |
if (super_check_offset.is_register()) { |
|
4480 |
local_jcc(Assembler::equal, *L_success); |
|
4481 |
cmpl(super_check_offset.as_register(), sc_offset); |
|
4482 |
if (L_failure == &L_fallthrough) { |
|
4483 |
local_jcc(Assembler::equal, *L_slow_path); |
|
4484 |
} else { |
|
4485 |
local_jcc(Assembler::notEqual, *L_failure); |
|
4486 |
final_jmp(*L_slow_path); |
|
4487 |
} |
|
4488 |
} else if (super_check_offset.as_constant() == sc_offset) { |
|
4489 |
// Need a slow path; fast failure is impossible. |
|
4490 |
if (L_slow_path == &L_fallthrough) { |
|
4491 |
local_jcc(Assembler::equal, *L_success); |
|
4492 |
} else { |
|
4493 |
local_jcc(Assembler::notEqual, *L_slow_path); |
|
4494 |
final_jmp(*L_success); |
|
4495 |
} |
|
4496 |
} else { |
|
4497 |
// No slow path; it's a fast decision. |
|
4498 |
if (L_failure == &L_fallthrough) { |
|
4499 |
local_jcc(Assembler::equal, *L_success); |
|
4500 |
} else { |
|
4501 |
local_jcc(Assembler::notEqual, *L_failure); |
|
4502 |
final_jmp(*L_success); |
|
4503 |
} |
|
4504 |
} |
|
4505 |
||
4506 |
bind(L_fallthrough); |
|
4507 |
||
4508 |
#undef local_jcc |
|
4509 |
#undef final_jmp |
|
4510 |
} |
|
4511 |
||
4512 |
||
4513 |
void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass, |
|
4514 |
Register super_klass, |
|
4515 |
Register temp_reg, |
|
4516 |
Register temp2_reg, |
|
4517 |
Label* L_success, |
|
4518 |
Label* L_failure, |
|
4519 |
bool set_cond_codes) { |
|
4520 |
assert_different_registers(sub_klass, super_klass, temp_reg); |
|
4521 |
if (temp2_reg != noreg) |
|
4522 |
assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg); |
|
4523 |
#define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg) |
|
4524 |
||
4525 |
Label L_fallthrough; |
|
4526 |
int label_nulls = 0; |
|
4527 |
if (L_success == NULL) { L_success = &L_fallthrough; label_nulls++; } |
|
4528 |
if (L_failure == NULL) { L_failure = &L_fallthrough; label_nulls++; } |
|
4529 |
assert(label_nulls <= 1, "at most one NULL in the batch"); |
|
4530 |
||
4531 |
// a couple of useful fields in sub_klass: |
|
4532 |
int ss_offset = in_bytes(Klass::secondary_supers_offset()); |
|
4533 |
int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); |
|
4534 |
Address secondary_supers_addr(sub_klass, ss_offset); |
|
4535 |
Address super_cache_addr( sub_klass, sc_offset); |
|
4536 |
||
4537 |
// Do a linear scan of the secondary super-klass chain. |
|
4538 |
// This code is rarely used, so simplicity is a virtue here. |
|
4539 |
// The repne_scan instruction uses fixed registers, which we must spill. |
|
4540 |
// Don't worry too much about pre-existing connections with the input regs. |
|
4541 |
||
4542 |
assert(sub_klass != rax, "killed reg"); // killed by mov(rax, super) |
|
4543 |
assert(sub_klass != rcx, "killed reg"); // killed by lea(rcx, &pst_counter) |
|
4544 |
||
4545 |
// Get super_klass value into rax (even if it was in rdi or rcx). |
|
4546 |
bool pushed_rax = false, pushed_rcx = false, pushed_rdi = false; |
|
4547 |
if (super_klass != rax || UseCompressedOops) { |
|
4548 |
if (!IS_A_TEMP(rax)) { push(rax); pushed_rax = true; } |
|
4549 |
mov(rax, super_klass); |
|
4550 |
} |
|
4551 |
if (!IS_A_TEMP(rcx)) { push(rcx); pushed_rcx = true; } |
|
4552 |
if (!IS_A_TEMP(rdi)) { push(rdi); pushed_rdi = true; } |
|
4553 |
||
4554 |
#ifndef PRODUCT |
|
4555 |
int* pst_counter = &SharedRuntime::_partial_subtype_ctr; |
|
4556 |
ExternalAddress pst_counter_addr((address) pst_counter); |
|
4557 |
NOT_LP64( incrementl(pst_counter_addr) ); |
|
4558 |
LP64_ONLY( lea(rcx, pst_counter_addr) ); |
|
4559 |
LP64_ONLY( incrementl(Address(rcx, 0)) ); |
|
4560 |
#endif //PRODUCT |
|
4561 |
||
4562 |
// We will consult the secondary-super array. |
|
4563 |
movptr(rdi, secondary_supers_addr); |
|
4564 |
// Load the array length. (Positive movl does right thing on LP64.) |
|
4565 |
movl(rcx, Address(rdi, Array<Klass*>::length_offset_in_bytes())); |
|
4566 |
// Skip to start of data. |
|
4567 |
addptr(rdi, Array<Klass*>::base_offset_in_bytes()); |
|
4568 |
||
4569 |
// Scan RCX words at [RDI] for an occurrence of RAX. |
|
4570 |
// Set NZ/Z based on last compare. |
|
4571 |
// Z flag value will not be set by 'repne' if RCX == 0 since 'repne' does |
|
4572 |
// not change flags (only scas instruction which is repeated sets flags). |
|
4573 |
// Set Z = 0 (not equal) before 'repne' to indicate that class was not found. |
|
4574 |
||
4575 |
testptr(rax,rax); // Set Z = 0 |
|
4576 |
repne_scan(); |
|
4577 |
||
4578 |
// Unspill the temp. registers: |
|
4579 |
if (pushed_rdi) pop(rdi); |
|
4580 |
if (pushed_rcx) pop(rcx); |
|
4581 |
if (pushed_rax) pop(rax); |
|
4582 |
||
4583 |
if (set_cond_codes) { |
|
4584 |
// Special hack for the AD files: rdi is guaranteed non-zero. |
|
4585 |
assert(!pushed_rdi, "rdi must be left non-NULL"); |
|
4586 |
// Also, the condition codes are properly set Z/NZ on succeed/failure. |
|
4587 |
} |
|
4588 |
||
4589 |
if (L_failure == &L_fallthrough) |
|
4590 |
jccb(Assembler::notEqual, *L_failure); |
|
4591 |
else jcc(Assembler::notEqual, *L_failure); |
|
4592 |
||
4593 |
// Success. Cache the super we found and proceed in triumph. |
|
4594 |
movptr(super_cache_addr, super_klass); |
|
4595 |
||
4596 |
if (L_success != &L_fallthrough) { |
|
4597 |
jmp(*L_success); |
|
4598 |
} |
|
4599 |
||
4600 |
#undef IS_A_TEMP |
|
4601 |
||
4602 |
bind(L_fallthrough); |
|
4603 |
} |
|
4604 |
||
55105
9ad765641e8f
8223213: Implement fast class initialization checks on x86-64
vlivanov
parents:
54839
diff
changeset
|
4605 |
void MacroAssembler::clinit_barrier(Register klass, Register thread, Label* L_fast_path, Label* L_slow_path) { |
9ad765641e8f
8223213: Implement fast class initialization checks on x86-64
vlivanov
parents:
54839
diff
changeset
|
4606 |
assert(L_fast_path != NULL || L_slow_path != NULL, "at least one is required"); |
9ad765641e8f
8223213: Implement fast class initialization checks on x86-64
vlivanov
parents:
54839
diff
changeset
|
4607 |
|
9ad765641e8f
8223213: Implement fast class initialization checks on x86-64
vlivanov
parents:
54839
diff
changeset
|
4608 |
Label L_fallthrough; |
9ad765641e8f
8223213: Implement fast class initialization checks on x86-64
vlivanov
parents:
54839
diff
changeset
|
4609 |
if (L_fast_path == NULL) { |
9ad765641e8f
8223213: Implement fast class initialization checks on x86-64
vlivanov
parents:
54839
diff
changeset
|
4610 |
L_fast_path = &L_fallthrough; |
55253
3c905e67e380
8225106: C2: Parse::clinit_deopt asserts when holder klass is in error state
vlivanov
parents:
55105
diff
changeset
|
4611 |
} else if (L_slow_path == NULL) { |
3c905e67e380
8225106: C2: Parse::clinit_deopt asserts when holder klass is in error state
vlivanov
parents:
55105
diff
changeset
|
4612 |
L_slow_path = &L_fallthrough; |
55105
9ad765641e8f
8223213: Implement fast class initialization checks on x86-64
vlivanov
parents:
54839
diff
changeset
|
4613 |
} |
9ad765641e8f
8223213: Implement fast class initialization checks on x86-64
vlivanov
parents:
54839
diff
changeset
|
4614 |
|
9ad765641e8f
8223213: Implement fast class initialization checks on x86-64
vlivanov
parents:
54839
diff
changeset
|
4615 |
// Fast path check: class is fully initialized |
9ad765641e8f
8223213: Implement fast class initialization checks on x86-64
vlivanov
parents:
54839
diff
changeset
|
4616 |
cmpb(Address(klass, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized); |
9ad765641e8f
8223213: Implement fast class initialization checks on x86-64
vlivanov
parents:
54839
diff
changeset
|
4617 |
jcc(Assembler::equal, *L_fast_path); |
9ad765641e8f
8223213: Implement fast class initialization checks on x86-64
vlivanov
parents:
54839
diff
changeset
|
4618 |
|
9ad765641e8f
8223213: Implement fast class initialization checks on x86-64
vlivanov
parents:
54839
diff
changeset
|
4619 |
// Fast path check: current thread is initializer thread |
9ad765641e8f
8223213: Implement fast class initialization checks on x86-64
vlivanov
parents:
54839
diff
changeset
|
4620 |
cmpptr(thread, Address(klass, InstanceKlass::init_thread_offset())); |
55253
3c905e67e380
8225106: C2: Parse::clinit_deopt asserts when holder klass is in error state
vlivanov
parents:
55105
diff
changeset
|
4621 |
if (L_slow_path == &L_fallthrough) { |
55105
9ad765641e8f
8223213: Implement fast class initialization checks on x86-64
vlivanov
parents:
54839
diff
changeset
|
4622 |
jcc(Assembler::equal, *L_fast_path); |
55253
3c905e67e380
8225106: C2: Parse::clinit_deopt asserts when holder klass is in error state
vlivanov
parents:
55105
diff
changeset
|
4623 |
bind(*L_slow_path); |
3c905e67e380
8225106: C2: Parse::clinit_deopt asserts when holder klass is in error state
vlivanov
parents:
55105
diff
changeset
|
4624 |
} else if (L_fast_path == &L_fallthrough) { |
3c905e67e380
8225106: C2: Parse::clinit_deopt asserts when holder klass is in error state
vlivanov
parents:
55105
diff
changeset
|
4625 |
jcc(Assembler::notEqual, *L_slow_path); |
3c905e67e380
8225106: C2: Parse::clinit_deopt asserts when holder klass is in error state
vlivanov
parents:
55105
diff
changeset
|
4626 |
bind(*L_fast_path); |
3c905e67e380
8225106: C2: Parse::clinit_deopt asserts when holder klass is in error state
vlivanov
parents:
55105
diff
changeset
|
4627 |
} else { |
3c905e67e380
8225106: C2: Parse::clinit_deopt asserts when holder klass is in error state
vlivanov
parents:
55105
diff
changeset
|
4628 |
Unimplemented(); |
3c905e67e380
8225106: C2: Parse::clinit_deopt asserts when holder klass is in error state
vlivanov
parents:
55105
diff
changeset
|
4629 |
} |
55105
9ad765641e8f
8223213: Implement fast class initialization checks on x86-64
vlivanov
parents:
54839
diff
changeset
|
4630 |
} |
9ad765641e8f
8223213: Implement fast class initialization checks on x86-64
vlivanov
parents:
54839
diff
changeset
|
4631 |
|
14626 | 4632 |
void MacroAssembler::cmov32(Condition cc, Register dst, Address src) { |
4633 |
if (VM_Version::supports_cmov()) { |
|
4634 |
cmovl(cc, dst, src); |
|
4635 |
} else { |
|
4636 |
Label L; |
|
4637 |
jccb(negate_condition(cc), L); |
|
4638 |
movl(dst, src); |
|
4639 |
bind(L); |
|
4640 |
} |
|
4641 |
} |
|
4642 |
||
4643 |
void MacroAssembler::cmov32(Condition cc, Register dst, Register src) { |
|
4644 |
if (VM_Version::supports_cmov()) { |
|
4645 |
cmovl(cc, dst, src); |
|
4646 |
} else { |
|
4647 |
Label L; |
|
4648 |
jccb(negate_condition(cc), L); |
|
4649 |
movl(dst, src); |
|
4650 |
bind(L); |
|
4651 |
} |
|
4652 |
} |
|
4653 |
||
4654 |
void MacroAssembler::verify_oop(Register reg, const char* s) { |
|
4655 |
if (!VerifyOops) return; |
|
4656 |
||
4657 |
// Pass register number to verify_oop_subroutine |
|
16368
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
4658 |
const char* b = NULL; |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
4659 |
{ |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
4660 |
ResourceMark rm; |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
4661 |
stringStream ss; |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
4662 |
ss.print("verify_oop: %s: %s", reg->name(), s); |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
4663 |
b = code_string(ss.as_string()); |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
4664 |
} |
14626 | 4665 |
BLOCK_COMMENT("verify_oop {"); |
4666 |
#ifdef _LP64 |
|
4667 |
push(rscratch1); // save r10, trashed by movptr() |
|
4668 |
#endif |
|
4669 |
push(rax); // save rax, |
|
4670 |
push(reg); // pass register argument |
|
4671 |
ExternalAddress buffer((address) b); |
|
4672 |
// avoid using pushptr, as it modifies scratch registers |
|
4673 |
// and our contract is not to modify anything |
|
4674 |
movptr(rax, buffer.addr()); |
|
4675 |
push(rax); |
|
4676 |
// call indirectly to solve generation ordering problem |
|
4677 |
movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address())); |
|
4678 |
call(rax); |
|
4679 |
// Caller pops the arguments (oop, message) and restores rax, r10 |
|
4680 |
BLOCK_COMMENT("} verify_oop"); |
|
4681 |
} |
|
4682 |
||
4683 |
||
4684 |
RegisterOrConstant MacroAssembler::delayed_value_impl(intptr_t* delayed_value_addr, |
|
4685 |
Register tmp, |
|
4686 |
int offset) { |
|
4687 |
intptr_t value = *delayed_value_addr; |
|
4688 |
if (value != 0) |
|
4689 |
return RegisterOrConstant(value + offset); |
|
4690 |
||
4691 |
// load indirectly to solve generation ordering problem |
|
4692 |
movptr(tmp, ExternalAddress((address) delayed_value_addr)); |
|
4693 |
||
4694 |
#ifdef ASSERT |
|
4695 |
{ Label L; |
|
4696 |
testptr(tmp, tmp); |
|
4697 |
if (WizardMode) { |
|
16368
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
4698 |
const char* buf = NULL; |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
4699 |
{ |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
4700 |
ResourceMark rm; |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
4701 |
stringStream ss; |
31592
43f48e165466
8081202: Hotspot compile warning: "Invalid suffix on literal; C++11 requires a space between literal and identifier"
bpittore
parents:
31369
diff
changeset
|
4702 |
ss.print("DelayedValue=" INTPTR_FORMAT, delayed_value_addr[1]); |
16368
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
4703 |
buf = code_string(ss.as_string()); |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
4704 |
} |
14626 | 4705 |
jcc(Assembler::notZero, L); |
4706 |
STOP(buf); |
|
4707 |
} else { |
|
4708 |
jccb(Assembler::notZero, L); |
|
4709 |
hlt(); |
|
4710 |
} |
|
4711 |
bind(L); |
|
4712 |
} |
|
4713 |
#endif |
|
4714 |
||
4715 |
if (offset != 0) |
|
4716 |
addptr(tmp, offset); |
|
4717 |
||
4718 |
return RegisterOrConstant(tmp); |
|
4719 |
} |
|
4720 |
||
4721 |
||
4722 |
Address MacroAssembler::argument_address(RegisterOrConstant arg_slot, |
|
4723 |
int extra_slot_offset) { |
|
4724 |
// cf. TemplateTable::prepare_invoke(), if (load_receiver). |
|
4725 |
int stackElementSize = Interpreter::stackElementSize; |
|
4726 |
int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0); |
|
4727 |
#ifdef ASSERT |
|
4728 |
int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1); |
|
4729 |
assert(offset1 - offset == stackElementSize, "correct arithmetic"); |
|
4730 |
#endif |
|
4731 |
Register scale_reg = noreg; |
|
4732 |
Address::ScaleFactor scale_factor = Address::no_scale; |
|
4733 |
if (arg_slot.is_constant()) { |
|
4734 |
offset += arg_slot.as_constant() * stackElementSize; |
|
4735 |
} else { |
|
4736 |
scale_reg = arg_slot.as_register(); |
|
4737 |
scale_factor = Address::times(stackElementSize); |
|
4738 |
} |
|
4739 |
offset += wordSize; // return PC is on stack |
|
4740 |
return Address(rsp, scale_reg, scale_factor, offset); |
|
4741 |
} |
|
4742 |
||
4743 |
||
4744 |
void MacroAssembler::verify_oop_addr(Address addr, const char* s) { |
|
4745 |
if (!VerifyOops) return; |
|
4746 |
||
4747 |
// Address adjust(addr.base(), addr.index(), addr.scale(), addr.disp() + BytesPerWord); |
|
4748 |
// Pass register number to verify_oop_subroutine |
|
16368
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
4749 |
const char* b = NULL; |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
4750 |
{ |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
4751 |
ResourceMark rm; |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
4752 |
stringStream ss; |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
4753 |
ss.print("verify_oop_addr: %s", s); |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
4754 |
b = code_string(ss.as_string()); |
713209c45a82
8008555: Debugging code in compiled method sometimes leaks memory
roland
parents:
15612
diff
changeset
|
4755 |
} |
14626 | 4756 |
#ifdef _LP64 |
4757 |
push(rscratch1); // save r10, trashed by movptr() |
|
4758 |
#endif |
|
4759 |
push(rax); // save rax, |
|
4760 |
// addr may contain rsp so we will have to adjust it based on the push |
|
4761 |
// we just did (and on 64 bit we do two pushes) |
|
4762 |
// NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which |
|
4763 |
// stores rax into addr which is backwards of what was intended. |
|
4764 |
if (addr.uses(rsp)) { |
|
4765 |
lea(rax, addr); |
|
4766 |
pushptr(Address(rax, LP64_ONLY(2 *) BytesPerWord)); |
|
4767 |
} else { |
|
4768 |
pushptr(addr); |
|
4769 |
} |
|
4770 |
||
4771 |
ExternalAddress buffer((address) b); |
|
4772 |
// pass msg argument |
|
4773 |
// avoid using pushptr, as it modifies scratch registers |
|
4774 |
// and our contract is not to modify anything |
|
4775 |
movptr(rax, buffer.addr()); |
|
4776 |
push(rax); |
|
4777 |
||
4778 |
// call indirectly to solve generation ordering problem |
|
4779 |
movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address())); |
|
4780 |
call(rax); |
|
4781 |
// Caller pops the arguments (addr, message) and restores rax, r10. |
|
4782 |
} |
|
4783 |
||
4784 |
void MacroAssembler::verify_tlab() { |
|
4785 |
#ifdef ASSERT |
|
4786 |
if (UseTLAB && VerifyOops) { |
|
4787 |
Label next, ok; |
|
4788 |
Register t1 = rsi; |
|
4789 |
Register thread_reg = NOT_LP64(rbx) LP64_ONLY(r15_thread); |
|
4790 |
||
4791 |
push(t1); |
|
4792 |
NOT_LP64(push(thread_reg)); |
|
4793 |
NOT_LP64(get_thread(thread_reg)); |
|
4794 |
||
4795 |
movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset()))); |
|
4796 |
cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset()))); |
|
4797 |
jcc(Assembler::aboveEqual, next); |
|
4798 |
STOP("assert(top >= start)"); |
|
4799 |
should_not_reach_here(); |
|
4800 |
||
4801 |
bind(next); |
|
4802 |
movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset()))); |
|
4803 |
cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset()))); |
|
4804 |
jcc(Assembler::aboveEqual, ok); |
|
4805 |
STOP("assert(top <= end)"); |
|
4806 |
should_not_reach_here(); |
|
4807 |
||
4808 |
bind(ok); |
|
4809 |
NOT_LP64(pop(thread_reg)); |
|
4810 |
pop(t1); |
|
4811 |
} |
|
4812 |
#endif |
|
4813 |
} |
|
4814 |
||
4815 |
class ControlWord { |
|
4816 |
public: |
|
4817 |
int32_t _value; |
|
4818 |
||
4819 |
int rounding_control() const { return (_value >> 10) & 3 ; } |
|
4820 |
int precision_control() const { return (_value >> 8) & 3 ; } |
|
4821 |
bool precision() const { return ((_value >> 5) & 1) != 0; } |
|
4822 |
bool underflow() const { return ((_value >> 4) & 1) != 0; } |
|
4823 |
bool overflow() const { return ((_value >> 3) & 1) != 0; } |
|
4824 |
bool zero_divide() const { return ((_value >> 2) & 1) != 0; } |
|
4825 |
bool denormalized() const { return ((_value >> 1) & 1) != 0; } |
|
4826 |
bool invalid() const { return ((_value >> 0) & 1) != 0; } |
|
4827 |
||
4828 |
void print() const { |
|
4829 |
// rounding control |
|
4830 |
const char* rc; |
|
4831 |
switch (rounding_control()) { |
|
4832 |
case 0: rc = "round near"; break; |
|
4833 |
case 1: rc = "round down"; break; |
|
4834 |
case 2: rc = "round up "; break; |
|
4835 |
case 3: rc = "chop "; break; |
|
4836 |
}; |
|
4837 |
// precision control |
|
4838 |
const char* pc; |
|
4839 |
switch (precision_control()) { |
|
4840 |
case 0: pc = "24 bits "; break; |
|
4841 |
case 1: pc = "reserved"; break; |
|
4842 |
case 2: pc = "53 bits "; break; |
|
4843 |
case 3: pc = "64 bits "; break; |
|
4844 |
}; |
|
4845 |
// flags |
|
4846 |
char f[9]; |
|
4847 |
f[0] = ' '; |
|
4848 |
f[1] = ' '; |
|
4849 |
f[2] = (precision ()) ? 'P' : 'p'; |
|
4850 |
f[3] = (underflow ()) ? 'U' : 'u'; |
|
4851 |
f[4] = (overflow ()) ? 'O' : 'o'; |
|
4852 |
f[5] = (zero_divide ()) ? 'Z' : 'z'; |
|
4853 |
f[6] = (denormalized()) ? 'D' : 'd'; |
|
4854 |
f[7] = (invalid ()) ? 'I' : 'i'; |
|
4855 |
f[8] = '\x0'; |
|
4856 |
// output |
|
4857 |
printf("%04x masks = %s, %s, %s", _value & 0xFFFF, f, rc, pc); |
|
4858 |
} |
|
4859 |
||
4860 |
}; |
|
4861 |
||
4862 |
class StatusWord { |
|
4863 |
public: |
|
4864 |
int32_t _value; |
|
4865 |
||
4866 |
bool busy() const { return ((_value >> 15) & 1) != 0; } |
|
4867 |
bool C3() const { return ((_value >> 14) & 1) != 0; } |
|
4868 |
bool C2() const { return ((_value >> 10) & 1) != 0; } |
|
4869 |
bool C1() const { return ((_value >> 9) & 1) != 0; } |
|
4870 |
bool C0() const { return ((_value >> 8) & 1) != 0; } |
|
4871 |
int top() const { return (_value >> 11) & 7 ; } |
|
4872 |
bool error_status() const { return ((_value >> 7) & 1) != 0; } |
|
4873 |
bool stack_fault() const { return ((_value >> 6) & 1) != 0; } |
|
4874 |
bool precision() const { return ((_value >> 5) & 1) != 0; } |
|
4875 |
bool underflow() const { return ((_value >> 4) & 1) != 0; } |
|
4876 |
bool overflow() const { return ((_value >> 3) & 1) != 0; } |
|
4877 |
bool zero_divide() const { return ((_value >> 2) & 1) != 0; } |
|
4878 |
bool denormalized() const { return ((_value >> 1) & 1) != 0; } |
|
4879 |
bool invalid() const { return ((_value >> 0) & 1) != 0; } |
|
4880 |
||
4881 |
void print() const { |
|
4882 |
// condition codes |
|
4883 |
char c[5]; |
|
4884 |
c[0] = (C3()) ? '3' : '-'; |
|
4885 |
c[1] = (C2()) ? '2' : '-'; |
|
4886 |
c[2] = (C1()) ? '1' : '-'; |
|
4887 |
c[3] = (C0()) ? '0' : '-'; |
|
4888 |
c[4] = '\x0'; |
|
4889 |
// flags |
|
4890 |
char f[9]; |
|
4891 |
f[0] = (error_status()) ? 'E' : '-'; |
|
4892 |
f[1] = (stack_fault ()) ? 'S' : '-'; |
|
4893 |
f[2] = (precision ()) ? 'P' : '-'; |
|
4894 |
f[3] = (underflow ()) ? 'U' : '-'; |
|
4895 |
f[4] = (overflow ()) ? 'O' : '-'; |
|
4896 |
f[5] = (zero_divide ()) ? 'Z' : '-'; |
|
4897 |
f[6] = (denormalized()) ? 'D' : '-'; |
|
4898 |
f[7] = (invalid ()) ? 'I' : '-'; |
|
4899 |
f[8] = '\x0'; |
|
4900 |
// output |
|
4901 |
printf("%04x flags = %s, cc = %s, top = %d", _value & 0xFFFF, f, c, top()); |
|
4902 |
} |
|
4903 |
||
4904 |
}; |
|
4905 |
||
4906 |
class TagWord { |
|
4907 |
public: |
|
4908 |
int32_t _value; |
|
4909 |
||
4910 |
int tag_at(int i) const { return (_value >> (i*2)) & 3; } |
|
4911 |
||
4912 |
void print() const { |
|
4913 |
printf("%04x", _value & 0xFFFF); |
|
4914 |
} |
|
4915 |
||
4916 |
}; |
|
4917 |
||
4918 |
class FPU_Register { |
|
4919 |
public: |
|
4920 |
int32_t _m0; |
|
4921 |
int32_t _m1; |
|
4922 |
int16_t _ex; |
|
4923 |
||
4924 |
bool is_indefinite() const { |
|
4925 |
return _ex == -1 && _m1 == (int32_t)0xC0000000 && _m0 == 0; |
|
4926 |
} |
|
4927 |
||
4928 |
void print() const { |
|
4929 |
char sign = (_ex < 0) ? '-' : '+'; |
|
4930 |
const char* kind = (_ex == 0x7FFF || _ex == (int16_t)-1) ? "NaN" : " "; |
|
4931 |
printf("%c%04hx.%08x%08x %s", sign, _ex, _m1, _m0, kind); |
|
4932 |
}; |
|
4933 |
||
4934 |
}; |
|
4935 |
||
4936 |
class FPU_State { |
|
4937 |
public: |
|
4938 |
enum { |
|
4939 |
register_size = 10, |
|
4940 |
number_of_registers = 8, |
|
4941 |
register_mask = 7 |
|
4942 |
}; |
|
4943 |
||
4944 |
ControlWord _control_word; |
|
4945 |
StatusWord _status_word; |
|
4946 |
TagWord _tag_word; |
|
4947 |
int32_t _error_offset; |
|
4948 |
int32_t _error_selector; |
|
4949 |
int32_t _data_offset; |
|
4950 |
int32_t _data_selector; |
|
4951 |
int8_t _register[register_size * number_of_registers]; |
|
4952 |
||
4953 |
int tag_for_st(int i) const { return _tag_word.tag_at((_status_word.top() + i) & register_mask); } |
|
4954 |
FPU_Register* st(int i) const { return (FPU_Register*)&_register[register_size * i]; } |
|
4955 |
||
4956 |
const char* tag_as_string(int tag) const { |
|
4957 |
switch (tag) { |
|
4958 |
case 0: return "valid"; |
|
4959 |
case 1: return "zero"; |
|
4960 |
case 2: return "special"; |
|
4961 |
case 3: return "empty"; |
|
4962 |
} |
|
4963 |
ShouldNotReachHere(); |
|
4964 |
return NULL; |
|
4965 |
} |
|
4966 |
||
4967 |
void print() const { |
|
4968 |
// print computation registers |
|
4969 |
{ int t = _status_word.top(); |
|
4970 |
for (int i = 0; i < number_of_registers; i++) { |
|
4971 |
int j = (i - t) & register_mask; |
|
4972 |
printf("%c r%d = ST%d = ", (j == 0 ? '*' : ' '), i, j); |
|
4973 |
st(j)->print(); |
|
4974 |
printf(" %s\n", tag_as_string(_tag_word.tag_at(i))); |
|
4975 |
} |
|
4976 |
} |
|
4977 |
printf("\n"); |
|
4978 |
// print control registers |
|
4979 |
printf("ctrl = "); _control_word.print(); printf("\n"); |
|
4980 |
printf("stat = "); _status_word .print(); printf("\n"); |
|
4981 |
printf("tags = "); _tag_word .print(); printf("\n"); |
|
4982 |
} |
|
4983 |
||
4984 |
}; |
|
4985 |
||
4986 |
class Flag_Register { |
|
4987 |
public: |
|
4988 |
int32_t _value; |
|
4989 |
||
4990 |
bool overflow() const { return ((_value >> 11) & 1) != 0; } |
|
4991 |
bool direction() const { return ((_value >> 10) & 1) != 0; } |
|
4992 |
bool sign() const { return ((_value >> 7) & 1) != 0; } |
|
4993 |
bool zero() const { return ((_value >> 6) & 1) != 0; } |
|
4994 |
bool auxiliary_carry() const { return ((_value >> 4) & 1) != 0; } |
|
4995 |
bool parity() const { return ((_value >> 2) & 1) != 0; } |
|
4996 |
bool carry() const { return ((_value >> 0) & 1) != 0; } |
|
4997 |
||
4998 |
void print() const { |
|
4999 |
// flags |
|
5000 |
char f[8]; |
|
5001 |
f[0] = (overflow ()) ? 'O' : '-'; |
|
5002 |
f[1] = (direction ()) ? 'D' : '-'; |
|
5003 |
f[2] = (sign ()) ? 'S' : '-'; |
|
5004 |
f[3] = (zero ()) ? 'Z' : '-'; |
|
5005 |
f[4] = (auxiliary_carry()) ? 'A' : '-'; |
|
5006 |
f[5] = (parity ()) ? 'P' : '-'; |
|
5007 |
f[6] = (carry ()) ? 'C' : '-'; |
|
5008 |
f[7] = '\x0'; |
|
5009 |
// output |
|
5010 |
printf("%08x flags = %s", _value, f); |
|
5011 |
} |
|
5012 |
||
5013 |
}; |
|
5014 |
||
5015 |
class IU_Register { |
|
5016 |
public: |
|
5017 |
int32_t _value; |
|
5018 |
||
5019 |
void print() const { |
|
5020 |
printf("%08x %11d", _value, _value); |
|
5021 |
} |
|
5022 |
||
5023 |
}; |
|
5024 |
||
5025 |
class IU_State { |
|
5026 |
public: |
|
5027 |
Flag_Register _eflags; |
|
5028 |
IU_Register _rdi; |
|
5029 |
IU_Register _rsi; |
|
5030 |
IU_Register _rbp; |
|
5031 |
IU_Register _rsp; |
|
5032 |
IU_Register _rbx; |
|
5033 |
IU_Register _rdx; |
|
5034 |
IU_Register _rcx; |
|
5035 |
IU_Register _rax; |
|
5036 |
||
5037 |
void print() const { |
|
5038 |
// computation registers |
|
5039 |
printf("rax, = "); _rax.print(); printf("\n"); |
|
5040 |
printf("rbx, = "); _rbx.print(); printf("\n"); |
|
5041 |
printf("rcx = "); _rcx.print(); printf("\n"); |
|
5042 |
printf("rdx = "); _rdx.print(); printf("\n"); |
|
5043 |
printf("rdi = "); _rdi.print(); printf("\n"); |
|
5044 |
printf("rsi = "); _rsi.print(); printf("\n"); |
|
5045 |
printf("rbp, = "); _rbp.print(); printf("\n"); |
|
5046 |
printf("rsp = "); _rsp.print(); printf("\n"); |
|
5047 |
printf("\n"); |
|
5048 |
// control registers |
|
5049 |
printf("flgs = "); _eflags.print(); printf("\n"); |
|
5050 |
} |
|
5051 |
}; |
|
5052 |
||
5053 |
||
5054 |
class CPU_State { |
|
5055 |
public: |
|
5056 |
FPU_State _fpu_state; |
|
5057 |
IU_State _iu_state; |
|
5058 |
||
5059 |
void print() const { |
|
5060 |
printf("--------------------------------------------------\n"); |
|
5061 |
_iu_state .print(); |
|
5062 |
printf("\n"); |
|
5063 |
_fpu_state.print(); |
|
5064 |
printf("--------------------------------------------------\n"); |
|
5065 |
} |
|
5066 |
||
5067 |
}; |
|
5068 |
||
5069 |
||
5070 |
static void _print_CPU_state(CPU_State* state) { |
|
5071 |
state->print(); |
|
5072 |
}; |
|
5073 |
||
5074 |
||
5075 |
void MacroAssembler::print_CPU_state() { |
|
5076 |
push_CPU_state(); |
|
5077 |
push(rsp); // pass CPU state |
|
5078 |
call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state))); |
|
5079 |
addptr(rsp, wordSize); // discard argument |
|
5080 |
pop_CPU_state(); |
|
5081 |
} |
|
5082 |
||
5083 |
||
5084 |
static bool _verify_FPU(int stack_depth, char* s, CPU_State* state) { |
|
5085 |
static int counter = 0; |
|
5086 |
FPU_State* fs = &state->_fpu_state; |
|
5087 |
counter++; |
|
5088 |
// For leaf calls, only verify that the top few elements remain empty. |
|
5089 |
// We only need 1 empty at the top for C2 code. |
|
5090 |
if( stack_depth < 0 ) { |
|
5091 |
if( fs->tag_for_st(7) != 3 ) { |
|
5092 |
printf("FPR7 not empty\n"); |
|
5093 |
state->print(); |
|
5094 |
assert(false, "error"); |
|
5095 |
return false; |
|
5096 |
} |
|
5097 |
return true; // All other stack states do not matter |
|
5098 |
} |
|
5099 |
||
5100 |
assert((fs->_control_word._value & 0xffff) == StubRoutines::_fpu_cntrl_wrd_std, |
|
5101 |
"bad FPU control word"); |
|
5102 |
||
5103 |
// compute stack depth |
|
5104 |
int i = 0; |
|
5105 |
while (i < FPU_State::number_of_registers && fs->tag_for_st(i) < 3) i++; |
|
5106 |
int d = i; |
|
5107 |
while (i < FPU_State::number_of_registers && fs->tag_for_st(i) == 3) i++; |
|
5108 |
// verify findings |
|
5109 |
if (i != FPU_State::number_of_registers) { |
|
5110 |
// stack not contiguous |
|
5111 |
printf("%s: stack not contiguous at ST%d\n", s, i); |
|
5112 |
state->print(); |
|
5113 |
assert(false, "error"); |
|
5114 |
return false; |
|
5115 |
} |
|
5116 |
// check if computed stack depth corresponds to expected stack depth |
|
5117 |
if (stack_depth < 0) { |
|
5118 |
// expected stack depth is -stack_depth or less |
|
5119 |
if (d > -stack_depth) { |
|
5120 |
// too many elements on the stack |
|
5121 |
printf("%s: <= %d stack elements expected but found %d\n", s, -stack_depth, d); |
|
5122 |
state->print(); |
|
5123 |
assert(false, "error"); |
|
5124 |
return false; |
|
5125 |
} |
|
5126 |
} else { |
|
5127 |
// expected stack depth is stack_depth |
|
5128 |
if (d != stack_depth) { |
|
5129 |
// wrong stack depth |
|
5130 |
printf("%s: %d stack elements expected but found %d\n", s, stack_depth, d); |
|
5131 |
state->print(); |
|
5132 |
assert(false, "error"); |
|
5133 |
return false; |
|
5134 |
} |
|
5135 |
} |
|
5136 |
// everything is cool |
|
5137 |
return true; |
|
5138 |
} |
|
5139 |
||
5140 |
||
5141 |
void MacroAssembler::verify_FPU(int stack_depth, const char* s) { |
|
5142 |
if (!VerifyFPU) return; |
|
5143 |
push_CPU_state(); |
|
5144 |
push(rsp); // pass CPU state |
|
5145 |
ExternalAddress msg((address) s); |
|
5146 |
// pass message string s |
|
5147 |
pushptr(msg.addr()); |
|
5148 |
push(stack_depth); // pass stack depth |
|
5149 |
call(RuntimeAddress(CAST_FROM_FN_PTR(address, _verify_FPU))); |
|
5150 |
addptr(rsp, 3 * wordSize); // discard arguments |
|
5151 |
// check for error |
|
5152 |
{ Label L; |
|
5153 |
testl(rax, rax); |
|
5154 |
jcc(Assembler::notZero, L); |
|
5155 |
int3(); // break if error condition |
|
5156 |
bind(L); |
|
5157 |
} |
|
5158 |
pop_CPU_state(); |
|
5159 |
} |
|
5160 |
||
16624
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
5161 |
void MacroAssembler::restore_cpu_control_state_after_jni() { |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
5162 |
// Either restore the MXCSR register after returning from the JNI Call |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
5163 |
// or verify that it wasn't changed (with -Xcheck:jni flag). |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
5164 |
if (VM_Version::supports_sse()) { |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
5165 |
if (RestoreMXCSROnJNICalls) { |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
5166 |
ldmxcsr(ExternalAddress(StubRoutines::addr_mxcsr_std())); |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
5167 |
} else if (CheckJNICalls) { |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
5168 |
call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry())); |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
5169 |
} |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
5170 |
} |
46440
61025eecb743
8178811: Minimize the AVX <-> SSE transition penalty through generation of vzeroupper instruction on x86
vdeshpande
parents:
44406
diff
changeset
|
5171 |
// Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty. |
61025eecb743
8178811: Minimize the AVX <-> SSE transition penalty through generation of vzeroupper instruction on x86
vdeshpande
parents:
44406
diff
changeset
|
5172 |
vzeroupper(); |
48194
09b7b32b244f
8190494: Different results with UseAVX=3 when calling AVX-512 native function via JNI
vdeshpande
parents:
47881
diff
changeset
|
5173 |
// Reset k1 to 0xffff. |
52003
be4614f04eb6
8211375: Minimal VM build failures after JDK-8211251 (Default mask register for avx512 instructions)
shade
parents:
51996
diff
changeset
|
5174 |
|
be4614f04eb6
8211375: Minimal VM build failures after JDK-8211251 (Default mask register for avx512 instructions)
shade
parents:
51996
diff
changeset
|
5175 |
#ifdef COMPILER2 |
51976
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
5176 |
if (PostLoopMultiversioning && VM_Version::supports_evex()) { |
48194
09b7b32b244f
8190494: Different results with UseAVX=3 when calling AVX-512 native function via JNI
vdeshpande
parents:
47881
diff
changeset
|
5177 |
push(rcx); |
09b7b32b244f
8190494: Different results with UseAVX=3 when calling AVX-512 native function via JNI
vdeshpande
parents:
47881
diff
changeset
|
5178 |
movl(rcx, 0xffff); |
09b7b32b244f
8190494: Different results with UseAVX=3 when calling AVX-512 native function via JNI
vdeshpande
parents:
47881
diff
changeset
|
5179 |
kmovwl(k1, rcx); |
09b7b32b244f
8190494: Different results with UseAVX=3 when calling AVX-512 native function via JNI
vdeshpande
parents:
47881
diff
changeset
|
5180 |
pop(rcx); |
09b7b32b244f
8190494: Different results with UseAVX=3 when calling AVX-512 native function via JNI
vdeshpande
parents:
47881
diff
changeset
|
5181 |
} |
52003
be4614f04eb6
8211375: Minimal VM build failures after JDK-8211251 (Default mask register for avx512 instructions)
shade
parents:
51996
diff
changeset
|
5182 |
#endif // COMPILER2 |
16624
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
5183 |
|
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
5184 |
#ifndef _LP64 |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
5185 |
// Either restore the x87 floating pointer control word after returning |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
5186 |
// from the JNI call or verify that it wasn't changed. |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
5187 |
if (CheckJNICalls) { |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
5188 |
call(RuntimeAddress(StubRoutines::x86::verify_fpu_cntrl_wrd_entry())); |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
5189 |
} |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
5190 |
#endif // _LP64 |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
5191 |
} |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
5192 |
|
46961
c9094b1e5f87
8186088: ConstantPoolCache::_resolved_references is not a JNIHandle
coleenp
parents:
46630
diff
changeset
|
5193 |
// ((OopHandle)result).resolve(); |
49816 | 5194 |
void MacroAssembler::resolve_oop_handle(Register result, Register tmp) { |
50157
bd198a98f3c5
8202978: Incorrect tmp register passed to MacroAssembler::load_mirror()
pliden
parents:
49902
diff
changeset
|
5195 |
assert_different_registers(result, tmp); |
bd198a98f3c5
8202978: Incorrect tmp register passed to MacroAssembler::load_mirror()
pliden
parents:
49902
diff
changeset
|
5196 |
|
49816 | 5197 |
// Only 64 bit platforms support GCs that require a tmp register |
5198 |
// Only IN_HEAP loads require a thread_tmp register |
|
5199 |
// OopHandle::resolve is an indirection like jobject. |
|
50803
45c1fde86050
8205559: Remove IN_CONCURRENT_ROOT Access decorator
kbarrett
parents:
50728
diff
changeset
|
5200 |
access_load_at(T_OBJECT, IN_NATIVE, |
49816 | 5201 |
result, Address(result, 0), tmp, /*tmp_thread*/noreg); |
5202 |
} |
|
5203 |
||
54839
e9db10a375d9
8222841: Incorrect static call stub interactions with class unloading
eosterlund
parents:
54780
diff
changeset
|
5204 |
// ((WeakHandle)result).resolve(); |
e9db10a375d9
8222841: Incorrect static call stub interactions with class unloading
eosterlund
parents:
54780
diff
changeset
|
5205 |
void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) { |
e9db10a375d9
8222841: Incorrect static call stub interactions with class unloading
eosterlund
parents:
54780
diff
changeset
|
5206 |
assert_different_registers(rresult, rtmp); |
e9db10a375d9
8222841: Incorrect static call stub interactions with class unloading
eosterlund
parents:
54780
diff
changeset
|
5207 |
Label resolved; |
e9db10a375d9
8222841: Incorrect static call stub interactions with class unloading
eosterlund
parents:
54780
diff
changeset
|
5208 |
|
e9db10a375d9
8222841: Incorrect static call stub interactions with class unloading
eosterlund
parents:
54780
diff
changeset
|
5209 |
// A null weak handle resolves to null. |
e9db10a375d9
8222841: Incorrect static call stub interactions with class unloading
eosterlund
parents:
54780
diff
changeset
|
5210 |
cmpptr(rresult, 0); |
e9db10a375d9
8222841: Incorrect static call stub interactions with class unloading
eosterlund
parents:
54780
diff
changeset
|
5211 |
jcc(Assembler::equal, resolved); |
e9db10a375d9
8222841: Incorrect static call stub interactions with class unloading
eosterlund
parents:
54780
diff
changeset
|
5212 |
|
e9db10a375d9
8222841: Incorrect static call stub interactions with class unloading
eosterlund
parents:
54780
diff
changeset
|
5213 |
// Only 64 bit platforms support GCs that require a tmp register |
e9db10a375d9
8222841: Incorrect static call stub interactions with class unloading
eosterlund
parents:
54780
diff
changeset
|
5214 |
// Only IN_HEAP loads require a thread_tmp register |
e9db10a375d9
8222841: Incorrect static call stub interactions with class unloading
eosterlund
parents:
54780
diff
changeset
|
5215 |
// WeakHandle::resolve is an indirection like jweak. |
e9db10a375d9
8222841: Incorrect static call stub interactions with class unloading
eosterlund
parents:
54780
diff
changeset
|
5216 |
access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, |
e9db10a375d9
8222841: Incorrect static call stub interactions with class unloading
eosterlund
parents:
54780
diff
changeset
|
5217 |
rresult, Address(rresult, 0), rtmp, /*tmp_thread*/noreg); |
e9db10a375d9
8222841: Incorrect static call stub interactions with class unloading
eosterlund
parents:
54780
diff
changeset
|
5218 |
bind(resolved); |
e9db10a375d9
8222841: Incorrect static call stub interactions with class unloading
eosterlund
parents:
54780
diff
changeset
|
5219 |
} |
e9db10a375d9
8222841: Incorrect static call stub interactions with class unloading
eosterlund
parents:
54780
diff
changeset
|
5220 |
|
49816 | 5221 |
void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) { |
38074
8475fdc6dcc3
8154580: Save mirror in interpreter frame to enable cleanups of CLDClosure
coleenp
parents:
38049
diff
changeset
|
5222 |
// get mirror |
8475fdc6dcc3
8154580: Save mirror in interpreter frame to enable cleanups of CLDClosure
coleenp
parents:
38049
diff
changeset
|
5223 |
const int mirror_offset = in_bytes(Klass::java_mirror_offset()); |
55105
9ad765641e8f
8223213: Implement fast class initialization checks on x86-64
vlivanov
parents:
54839
diff
changeset
|
5224 |
load_method_holder(mirror, method); |
38074
8475fdc6dcc3
8154580: Save mirror in interpreter frame to enable cleanups of CLDClosure
coleenp
parents:
38049
diff
changeset
|
5225 |
movptr(mirror, Address(mirror, mirror_offset)); |
49816 | 5226 |
resolve_oop_handle(mirror, tmp); |
38074
8475fdc6dcc3
8154580: Save mirror in interpreter frame to enable cleanups of CLDClosure
coleenp
parents:
38049
diff
changeset
|
5227 |
} |
16624
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
5228 |
|
54839
e9db10a375d9
8222841: Incorrect static call stub interactions with class unloading
eosterlund
parents:
54780
diff
changeset
|
5229 |
void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) { |
55105
9ad765641e8f
8223213: Implement fast class initialization checks on x86-64
vlivanov
parents:
54839
diff
changeset
|
5230 |
load_method_holder(rresult, rmethod); |
54839
e9db10a375d9
8222841: Incorrect static call stub interactions with class unloading
eosterlund
parents:
54780
diff
changeset
|
5231 |
movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset())); |
e9db10a375d9
8222841: Incorrect static call stub interactions with class unloading
eosterlund
parents:
54780
diff
changeset
|
5232 |
} |
e9db10a375d9
8222841: Incorrect static call stub interactions with class unloading
eosterlund
parents:
54780
diff
changeset
|
5233 |
|
55105
9ad765641e8f
8223213: Implement fast class initialization checks on x86-64
vlivanov
parents:
54839
diff
changeset
|
5234 |
void MacroAssembler::load_method_holder(Register holder, Register method) { |
9ad765641e8f
8223213: Implement fast class initialization checks on x86-64
vlivanov
parents:
54839
diff
changeset
|
5235 |
movptr(holder, Address(method, Method::const_offset())); // ConstMethod* |
9ad765641e8f
8223213: Implement fast class initialization checks on x86-64
vlivanov
parents:
54839
diff
changeset
|
5236 |
movptr(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool* |
9ad765641e8f
8223213: Implement fast class initialization checks on x86-64
vlivanov
parents:
54839
diff
changeset
|
5237 |
movptr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass* |
9ad765641e8f
8223213: Implement fast class initialization checks on x86-64
vlivanov
parents:
54839
diff
changeset
|
5238 |
} |
9ad765641e8f
8223213: Implement fast class initialization checks on x86-64
vlivanov
parents:
54839
diff
changeset
|
5239 |
|
14626 | 5240 |
void MacroAssembler::load_klass(Register dst, Register src) { |
5241 |
#ifdef _LP64 |
|
19979
ebe1dbb6e1aa
8015107: NPG: Use consistent naming for metaspace concepts
ehelin
parents:
19319
diff
changeset
|
5242 |
if (UseCompressedClassPointers) { |
14626 | 5243 |
movl(dst, Address(src, oopDesc::klass_offset_in_bytes())); |
5244 |
decode_klass_not_null(dst); |
|
5245 |
} else |
|
5246 |
#endif |
|
5247 |
movptr(dst, Address(src, oopDesc::klass_offset_in_bytes())); |
|
5248 |
} |
|
5249 |
||
5250 |
void MacroAssembler::load_prototype_header(Register dst, Register src) { |
|
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5251 |
load_klass(dst, src); |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5252 |
movptr(dst, Address(dst, Klass::prototype_header_offset())); |
14626 | 5253 |
} |
5254 |
||
5255 |
void MacroAssembler::store_klass(Register dst, Register src) { |
|
5256 |
#ifdef _LP64 |
|
19979
ebe1dbb6e1aa
8015107: NPG: Use consistent naming for metaspace concepts
ehelin
parents:
19319
diff
changeset
|
5257 |
if (UseCompressedClassPointers) { |
14626 | 5258 |
encode_klass_not_null(src); |
5259 |
movl(Address(dst, oopDesc::klass_offset_in_bytes()), src); |
|
5260 |
} else |
|
5261 |
#endif |
|
5262 |
movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src); |
|
5263 |
} |
|
5264 |
||
49748 | 5265 |
void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src, |
5266 |
Register tmp1, Register thread_tmp) { |
|
5267 |
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); |
|
50446
39ca7558bc43
8203353: Fixup inferred decorators in the interpreter
eosterlund
parents:
50157
diff
changeset
|
5268 |
decorators = AccessInternal::decorator_fixup(decorators); |
49748 | 5269 |
bool as_raw = (decorators & AS_RAW) != 0; |
5270 |
if (as_raw) { |
|
5271 |
bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp); |
|
5272 |
} else { |
|
5273 |
bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp); |
|
5274 |
} |
|
5275 |
} |
|
5276 |
||
5277 |
void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register src, |
|
5278 |
Register tmp1, Register tmp2) { |
|
5279 |
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); |
|
50446
39ca7558bc43
8203353: Fixup inferred decorators in the interpreter
eosterlund
parents:
50157
diff
changeset
|
5280 |
decorators = AccessInternal::decorator_fixup(decorators); |
49748 | 5281 |
bool as_raw = (decorators & AS_RAW) != 0; |
5282 |
if (as_raw) { |
|
5283 |
bs->BarrierSetAssembler::store_at(this, decorators, type, dst, src, tmp1, tmp2); |
|
5284 |
} else { |
|
5285 |
bs->store_at(this, decorators, type, dst, src, tmp1, tmp2); |
|
5286 |
} |
|
5287 |
} |
|
5288 |
||
51350 | 5289 |
void MacroAssembler::resolve(DecoratorSet decorators, Register obj) { |
5290 |
// Use stronger ACCESS_WRITE|ACCESS_READ by default. |
|
5291 |
if ((decorators & (ACCESS_READ | ACCESS_WRITE)) == 0) { |
|
5292 |
decorators |= ACCESS_READ | ACCESS_WRITE; |
|
5293 |
} |
|
5294 |
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); |
|
5295 |
return bs->resolve(this, decorators, obj); |
|
5296 |
} |
|
5297 |
||
49748 | 5298 |
void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1, |
5299 |
Register thread_tmp, DecoratorSet decorators) { |
|
5300 |
access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp); |
|
14626 | 5301 |
} |
5302 |
||
5303 |
// Doesn't do verfication, generates fixed size code |
|
49748 | 5304 |
void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1, |
5305 |
Register thread_tmp, DecoratorSet decorators) { |
|
50728 | 5306 |
access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp); |
49748 | 5307 |
} |
5308 |
||
5309 |
void MacroAssembler::store_heap_oop(Address dst, Register src, Register tmp1, |
|
5310 |
Register tmp2, DecoratorSet decorators) { |
|
5311 |
access_store_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, tmp2); |
|
14626 | 5312 |
} |
5313 |
||
5314 |
// Used for storing NULLs. |
|
5315 |
void MacroAssembler::store_heap_oop_null(Address dst) { |
|
49748 | 5316 |
access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg); |
14626 | 5317 |
} |
5318 |
||
5319 |
#ifdef _LP64 |
|
5320 |
void MacroAssembler::store_klass_gap(Register dst, Register src) { |
|
19979
ebe1dbb6e1aa
8015107: NPG: Use consistent naming for metaspace concepts
ehelin
parents:
19319
diff
changeset
|
5321 |
if (UseCompressedClassPointers) { |
14626 | 5322 |
// Store to klass gap in destination |
5323 |
movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src); |
|
5324 |
} |
|
5325 |
} |
|
5326 |
||
5327 |
#ifdef ASSERT |
|
5328 |
void MacroAssembler::verify_heapbase(const char* msg) { |
|
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5329 |
assert (UseCompressedOops, "should be compressed"); |
14626 | 5330 |
assert (Universe::heap() != NULL, "java heap should be initialized"); |
5331 |
if (CheckCompressedOops) { |
|
5332 |
Label ok; |
|
5333 |
push(rscratch1); // cmpptr trashes rscratch1 |
|
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5334 |
cmpptr(r12_heapbase, ExternalAddress((address)CompressedOops::ptrs_base_addr())); |
14626 | 5335 |
jcc(Assembler::equal, ok); |
5336 |
STOP(msg); |
|
5337 |
bind(ok); |
|
5338 |
pop(rscratch1); |
|
5339 |
} |
|
5340 |
} |
|
5341 |
#endif |
|
5342 |
||
5343 |
// Algorithm must match oop.inline.hpp encode_heap_oop. |
|
5344 |
void MacroAssembler::encode_heap_oop(Register r) { |
|
5345 |
#ifdef ASSERT |
|
5346 |
verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?"); |
|
5347 |
#endif |
|
5348 |
verify_oop(r, "broken oop in encode_heap_oop"); |
|
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5349 |
if (CompressedOops::base() == NULL) { |
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5350 |
if (CompressedOops::shift() != 0) { |
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5351 |
assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); |
14626 | 5352 |
shrq(r, LogMinObjAlignmentInBytes); |
5353 |
} |
|
5354 |
return; |
|
5355 |
} |
|
5356 |
testq(r, r); |
|
5357 |
cmovq(Assembler::equal, r, r12_heapbase); |
|
5358 |
subq(r, r12_heapbase); |
|
5359 |
shrq(r, LogMinObjAlignmentInBytes); |
|
5360 |
} |
|
5361 |
||
5362 |
void MacroAssembler::encode_heap_oop_not_null(Register r) { |
|
5363 |
#ifdef ASSERT |
|
5364 |
verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?"); |
|
5365 |
if (CheckCompressedOops) { |
|
5366 |
Label ok; |
|
5367 |
testq(r, r); |
|
5368 |
jcc(Assembler::notEqual, ok); |
|
5369 |
STOP("null oop passed to encode_heap_oop_not_null"); |
|
5370 |
bind(ok); |
|
5371 |
} |
|
5372 |
#endif |
|
5373 |
verify_oop(r, "broken oop in encode_heap_oop_not_null"); |
|
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5374 |
if (CompressedOops::base() != NULL) { |
14626 | 5375 |
subq(r, r12_heapbase); |
5376 |
} |
|
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5377 |
if (CompressedOops::shift() != 0) { |
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5378 |
assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); |
14626 | 5379 |
shrq(r, LogMinObjAlignmentInBytes); |
5380 |
} |
|
5381 |
} |
|
5382 |
||
5383 |
void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) { |
|
5384 |
#ifdef ASSERT |
|
5385 |
verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?"); |
|
5386 |
if (CheckCompressedOops) { |
|
5387 |
Label ok; |
|
5388 |
testq(src, src); |
|
5389 |
jcc(Assembler::notEqual, ok); |
|
5390 |
STOP("null oop passed to encode_heap_oop_not_null2"); |
|
5391 |
bind(ok); |
|
5392 |
} |
|
5393 |
#endif |
|
5394 |
verify_oop(src, "broken oop in encode_heap_oop_not_null2"); |
|
5395 |
if (dst != src) { |
|
5396 |
movq(dst, src); |
|
5397 |
} |
|
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5398 |
if (CompressedOops::base() != NULL) { |
14626 | 5399 |
subq(dst, r12_heapbase); |
5400 |
} |
|
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5401 |
if (CompressedOops::shift() != 0) { |
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5402 |
assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); |
14626 | 5403 |
shrq(dst, LogMinObjAlignmentInBytes); |
5404 |
} |
|
5405 |
} |
|
5406 |
||
5407 |
void MacroAssembler::decode_heap_oop(Register r) { |
|
5408 |
#ifdef ASSERT |
|
5409 |
verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?"); |
|
5410 |
#endif |
|
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5411 |
if (CompressedOops::base() == NULL) { |
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5412 |
if (CompressedOops::shift() != 0) { |
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5413 |
assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); |
14626 | 5414 |
shlq(r, LogMinObjAlignmentInBytes); |
5415 |
} |
|
5416 |
} else { |
|
5417 |
Label done; |
|
5418 |
shlq(r, LogMinObjAlignmentInBytes); |
|
5419 |
jccb(Assembler::equal, done); |
|
5420 |
addq(r, r12_heapbase); |
|
5421 |
bind(done); |
|
5422 |
} |
|
5423 |
verify_oop(r, "broken oop in decode_heap_oop"); |
|
5424 |
} |
|
5425 |
||
5426 |
void MacroAssembler::decode_heap_oop_not_null(Register r) { |
|
5427 |
// Note: it will change flags |
|
5428 |
assert (UseCompressedOops, "should only be used for compressed headers"); |
|
5429 |
assert (Universe::heap() != NULL, "java heap should be initialized"); |
|
5430 |
// Cannot assert, unverified entry point counts instructions (see .ad file) |
|
5431 |
// vtableStubs also counts instructions in pd_code_size_limit. |
|
5432 |
// Also do not verify_oop as this is called by verify_oop. |
|
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5433 |
if (CompressedOops::shift() != 0) { |
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5434 |
assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); |
14626 | 5435 |
shlq(r, LogMinObjAlignmentInBytes); |
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5436 |
if (CompressedOops::base() != NULL) { |
14626 | 5437 |
addq(r, r12_heapbase); |
5438 |
} |
|
5439 |
} else { |
|
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5440 |
assert (CompressedOops::base() == NULL, "sanity"); |
14626 | 5441 |
} |
5442 |
} |
|
5443 |
||
5444 |
void MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) { |
|
5445 |
// Note: it will change flags |
|
5446 |
assert (UseCompressedOops, "should only be used for compressed headers"); |
|
5447 |
assert (Universe::heap() != NULL, "java heap should be initialized"); |
|
5448 |
// Cannot assert, unverified entry point counts instructions (see .ad file) |
|
5449 |
// vtableStubs also counts instructions in pd_code_size_limit. |
|
5450 |
// Also do not verify_oop as this is called by verify_oop. |
|
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5451 |
if (CompressedOops::shift() != 0) { |
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5452 |
assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); |
14626 | 5453 |
if (LogMinObjAlignmentInBytes == Address::times_8) { |
5454 |
leaq(dst, Address(r12_heapbase, src, Address::times_8, 0)); |
|
5455 |
} else { |
|
5456 |
if (dst != src) { |
|
5457 |
movq(dst, src); |
|
5458 |
} |
|
5459 |
shlq(dst, LogMinObjAlignmentInBytes); |
|
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5460 |
if (CompressedOops::base() != NULL) { |
14626 | 5461 |
addq(dst, r12_heapbase); |
5462 |
} |
|
5463 |
} |
|
5464 |
} else { |
|
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5465 |
assert (CompressedOops::base() == NULL, "sanity"); |
14626 | 5466 |
if (dst != src) { |
5467 |
movq(dst, src); |
|
5468 |
} |
|
5469 |
} |
|
5470 |
} |
|
5471 |
||
5472 |
void MacroAssembler::encode_klass_not_null(Register r) { |
|
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5473 |
if (CompressedKlassPointers::base() != NULL) { |
21188
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
5474 |
// Use r12 as a scratch register in which to temporarily load the narrow_klass_base. |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
5475 |
assert(r != r12_heapbase, "Encoding a klass in r12"); |
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5476 |
mov64(r12_heapbase, (int64_t)CompressedKlassPointers::base()); |
21188
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
5477 |
subq(r, r12_heapbase); |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
5478 |
} |
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5479 |
if (CompressedKlassPointers::shift() != 0) { |
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5480 |
assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); |
14626 | 5481 |
shrq(r, LogKlassAlignmentInBytes); |
5482 |
} |
|
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5483 |
if (CompressedKlassPointers::base() != NULL) { |
21188
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
5484 |
reinit_heapbase(); |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
5485 |
} |
14626 | 5486 |
} |
5487 |
||
5488 |
void MacroAssembler::encode_klass_not_null(Register dst, Register src) { |
|
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5489 |
if (dst == src) { |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5490 |
encode_klass_not_null(src); |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5491 |
} else { |
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5492 |
if (CompressedKlassPointers::base() != NULL) { |
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5493 |
mov64(dst, (int64_t)CompressedKlassPointers::base()); |
21188
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
5494 |
negq(dst); |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
5495 |
addq(dst, src); |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
5496 |
} else { |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
5497 |
movptr(dst, src); |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
5498 |
} |
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5499 |
if (CompressedKlassPointers::shift() != 0) { |
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5500 |
assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); |
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5501 |
shrq(dst, LogKlassAlignmentInBytes); |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5502 |
} |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5503 |
} |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5504 |
} |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5505 |
|
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5506 |
// Function instr_size_for_decode_klass_not_null() counts the instructions |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5507 |
// generated by decode_klass_not_null(register r) and reinit_heapbase(), |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5508 |
// when (Universe::heap() != NULL). Hence, if the instructions they |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5509 |
// generate change, then this method needs to be updated. |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5510 |
int MacroAssembler::instr_size_for_decode_klass_not_null() { |
19979
ebe1dbb6e1aa
8015107: NPG: Use consistent naming for metaspace concepts
ehelin
parents:
19319
diff
changeset
|
5511 |
assert (UseCompressedClassPointers, "only for compressed klass ptrs"); |
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5512 |
if (CompressedKlassPointers::base() != NULL) { |
21188
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
5513 |
// mov64 + addq + shlq? + mov64 (for reinit_heapbase()). |
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5514 |
return (CompressedKlassPointers::shift() == 0 ? 20 : 24); |
21188
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
5515 |
} else { |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
5516 |
// longest load decode klass function, mov64, leaq |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
5517 |
return 16; |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
5518 |
} |
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5519 |
} |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5520 |
|
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5521 |
// !!! If the instructions that get generated here change then function |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5522 |
// instr_size_for_decode_klass_not_null() needs to get updated. |
14626 | 5523 |
void MacroAssembler::decode_klass_not_null(Register r) { |
5524 |
// Note: it will change flags |
|
19979
ebe1dbb6e1aa
8015107: NPG: Use consistent naming for metaspace concepts
ehelin
parents:
19319
diff
changeset
|
5525 |
assert (UseCompressedClassPointers, "should only be used for compressed headers"); |
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5526 |
assert(r != r12_heapbase, "Decoding a klass in r12"); |
14626 | 5527 |
// Cannot assert, unverified entry point counts instructions (see .ad file) |
5528 |
// vtableStubs also counts instructions in pd_code_size_limit. |
|
5529 |
// Also do not verify_oop as this is called by verify_oop. |
|
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5530 |
if (CompressedKlassPointers::shift() != 0) { |
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5531 |
assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); |
14626 | 5532 |
shlq(r, LogKlassAlignmentInBytes); |
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5533 |
} |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5534 |
// Use r12 as a scratch register in which to temporarily load the narrow_klass_base. |
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5535 |
if (CompressedKlassPointers::base() != NULL) { |
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5536 |
mov64(r12_heapbase, (int64_t)CompressedKlassPointers::base()); |
21188
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
5537 |
addq(r, r12_heapbase); |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
5538 |
reinit_heapbase(); |
d053e4e8f901
8024927: Nashorn performance regression with CompressedOops
coleenp
parents:
20403
diff
changeset
|
5539 |
} |
14626 | 5540 |
} |
5541 |
||
5542 |
void MacroAssembler::decode_klass_not_null(Register dst, Register src) { |
|
5543 |
// Note: it will change flags |
|
19979
ebe1dbb6e1aa
8015107: NPG: Use consistent naming for metaspace concepts
ehelin
parents:
19319
diff
changeset
|
5544 |
assert (UseCompressedClassPointers, "should only be used for compressed headers"); |
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5545 |
if (dst == src) { |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5546 |
decode_klass_not_null(dst); |
14626 | 5547 |
} else { |
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5548 |
// Cannot assert, unverified entry point counts instructions (see .ad file) |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5549 |
// vtableStubs also counts instructions in pd_code_size_limit. |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5550 |
// Also do not verify_oop as this is called by verify_oop. |
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5551 |
mov64(dst, (int64_t)CompressedKlassPointers::base()); |
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5552 |
if (CompressedKlassPointers::shift() != 0) { |
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5553 |
assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); |
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5554 |
assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?"); |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5555 |
leaq(dst, Address(dst, src, Address::times_8, 0)); |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5556 |
} else { |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5557 |
addq(dst, src); |
14626 | 5558 |
} |
5559 |
} |
|
5560 |
} |
|
5561 |
||
5562 |
void MacroAssembler::set_narrow_oop(Register dst, jobject obj) { |
|
5563 |
assert (UseCompressedOops, "should only be used for compressed headers"); |
|
5564 |
assert (Universe::heap() != NULL, "java heap should be initialized"); |
|
5565 |
assert (oop_recorder() != NULL, "this assembler needs an OopRecorder"); |
|
5566 |
int oop_index = oop_recorder()->find_index(obj); |
|
5567 |
RelocationHolder rspec = oop_Relocation::spec(oop_index); |
|
5568 |
mov_narrow_oop(dst, oop_index, rspec); |
|
5569 |
} |
|
5570 |
||
5571 |
void MacroAssembler::set_narrow_oop(Address dst, jobject obj) { |
|
5572 |
assert (UseCompressedOops, "should only be used for compressed headers"); |
|
5573 |
assert (Universe::heap() != NULL, "java heap should be initialized"); |
|
5574 |
assert (oop_recorder() != NULL, "this assembler needs an OopRecorder"); |
|
5575 |
int oop_index = oop_recorder()->find_index(obj); |
|
5576 |
RelocationHolder rspec = oop_Relocation::spec(oop_index); |
|
5577 |
mov_narrow_oop(dst, oop_index, rspec); |
|
5578 |
} |
|
5579 |
||
5580 |
void MacroAssembler::set_narrow_klass(Register dst, Klass* k) { |
|
19979
ebe1dbb6e1aa
8015107: NPG: Use consistent naming for metaspace concepts
ehelin
parents:
19319
diff
changeset
|
5581 |
assert (UseCompressedClassPointers, "should only be used for compressed headers"); |
14626 | 5582 |
assert (oop_recorder() != NULL, "this assembler needs an OopRecorder"); |
5583 |
int klass_index = oop_recorder()->find_index(k); |
|
5584 |
RelocationHolder rspec = metadata_Relocation::spec(klass_index); |
|
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5585 |
mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec); |
14626 | 5586 |
} |
5587 |
||
5588 |
void MacroAssembler::set_narrow_klass(Address dst, Klass* k) { |
|
19979
ebe1dbb6e1aa
8015107: NPG: Use consistent naming for metaspace concepts
ehelin
parents:
19319
diff
changeset
|
5589 |
assert (UseCompressedClassPointers, "should only be used for compressed headers"); |
14626 | 5590 |
assert (oop_recorder() != NULL, "this assembler needs an OopRecorder"); |
5591 |
int klass_index = oop_recorder()->find_index(k); |
|
5592 |
RelocationHolder rspec = metadata_Relocation::spec(klass_index); |
|
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5593 |
mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec); |
14626 | 5594 |
} |
5595 |
||
5596 |
void MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) { |
|
5597 |
assert (UseCompressedOops, "should only be used for compressed headers"); |
|
5598 |
assert (Universe::heap() != NULL, "java heap should be initialized"); |
|
5599 |
assert (oop_recorder() != NULL, "this assembler needs an OopRecorder"); |
|
5600 |
int oop_index = oop_recorder()->find_index(obj); |
|
5601 |
RelocationHolder rspec = oop_Relocation::spec(oop_index); |
|
5602 |
Assembler::cmp_narrow_oop(dst, oop_index, rspec); |
|
5603 |
} |
|
5604 |
||
5605 |
void MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) { |
|
5606 |
assert (UseCompressedOops, "should only be used for compressed headers"); |
|
5607 |
assert (Universe::heap() != NULL, "java heap should be initialized"); |
|
5608 |
assert (oop_recorder() != NULL, "this assembler needs an OopRecorder"); |
|
5609 |
int oop_index = oop_recorder()->find_index(obj); |
|
5610 |
RelocationHolder rspec = oop_Relocation::spec(oop_index); |
|
5611 |
Assembler::cmp_narrow_oop(dst, oop_index, rspec); |
|
5612 |
} |
|
5613 |
||
5614 |
void MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) { |
|
19979
ebe1dbb6e1aa
8015107: NPG: Use consistent naming for metaspace concepts
ehelin
parents:
19319
diff
changeset
|
5615 |
assert (UseCompressedClassPointers, "should only be used for compressed headers"); |
14626 | 5616 |
assert (oop_recorder() != NULL, "this assembler needs an OopRecorder"); |
5617 |
int klass_index = oop_recorder()->find_index(k); |
|
5618 |
RelocationHolder rspec = metadata_Relocation::spec(klass_index); |
|
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5619 |
Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec); |
14626 | 5620 |
} |
5621 |
||
5622 |
void MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) { |
|
19979
ebe1dbb6e1aa
8015107: NPG: Use consistent naming for metaspace concepts
ehelin
parents:
19319
diff
changeset
|
5623 |
assert (UseCompressedClassPointers, "should only be used for compressed headers"); |
14626 | 5624 |
assert (oop_recorder() != NULL, "this assembler needs an OopRecorder"); |
5625 |
int klass_index = oop_recorder()->find_index(k); |
|
5626 |
RelocationHolder rspec = metadata_Relocation::spec(klass_index); |
|
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5627 |
Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec); |
14626 | 5628 |
} |
5629 |
||
5630 |
void MacroAssembler::reinit_heapbase() { |
|
19979
ebe1dbb6e1aa
8015107: NPG: Use consistent naming for metaspace concepts
ehelin
parents:
19319
diff
changeset
|
5631 |
if (UseCompressedOops || UseCompressedClassPointers) { |
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5632 |
if (Universe::heap() != NULL) { |
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5633 |
if (CompressedOops::base() == NULL) { |
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5634 |
MacroAssembler::xorptr(r12_heapbase, r12_heapbase); |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5635 |
} else { |
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5636 |
mov64(r12_heapbase, (int64_t)CompressedOops::ptrs_base()); |
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5637 |
} |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5638 |
} else { |
54780
f8d182aedc92
8223136: Move compressed oops functions to CompressedOops class
stefank
parents:
54750
diff
changeset
|
5639 |
movptr(r12_heapbase, ExternalAddress((address)CompressedOops::ptrs_base_addr())); |
19319
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5640 |
} |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5641 |
} |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5642 |
} |
0ad35be0733a
8003424: Enable Class Data Sharing for CompressedOops
hseigel
parents:
18507
diff
changeset
|
5643 |
|
14626 | 5644 |
#endif // _LP64 |
5645 |
||
5646 |
// C2 compiled method's prolog code. |
|
52142 | 5647 |
void MacroAssembler::verified_entry(int framesize, int stack_bang_size, bool fp_mode_24b, bool is_stub) { |
14626 | 5648 |
|
5649 |
// WARNING: Initial instruction MUST be 5 bytes or longer so that |
|
5650 |
// NativeJump::patch_verified_entry will be able to patch out the entry |
|
5651 |
// code safely. The push to verify stack depth is ok at 5 bytes, |
|
5652 |
// the frame allocation can be either 3 or 6 bytes. So if we don't do |
|
5653 |
// stack bang then we must use the 6 byte frame allocation even if |
|
5654 |
// we have no frame. :-( |
|
24018
77b156916bab
8032410: compiler/uncommontrap/TestStackBangRbp.java times out on Solaris-Sparc V9
roland
parents:
23847
diff
changeset
|
5655 |
assert(stack_bang_size >= framesize || stack_bang_size <= 0, "stack bang size incorrect"); |
14626 | 5656 |
|
5657 |
assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned"); |
|
5658 |
// Remove word for return addr |
|
5659 |
framesize -= wordSize; |
|
24018
77b156916bab
8032410: compiler/uncommontrap/TestStackBangRbp.java times out on Solaris-Sparc V9
roland
parents:
23847
diff
changeset
|
5660 |
stack_bang_size -= wordSize; |
14626 | 5661 |
|
5662 |
// Calls to C2R adapters often do not accept exceptional returns. |
|
5663 |
// We require that their callers must bang for them. But be careful, because |
|
5664 |
// some VM calls (such as call site linkage) can use several kilobytes of |
|
5665 |
// stack. But the stack safety zone should account for that. |
|
5666 |
// See bugs 4446381, 4468289, 4497237. |
|
24018
77b156916bab
8032410: compiler/uncommontrap/TestStackBangRbp.java times out on Solaris-Sparc V9
roland
parents:
23847
diff
changeset
|
5667 |
if (stack_bang_size > 0) { |
77b156916bab
8032410: compiler/uncommontrap/TestStackBangRbp.java times out on Solaris-Sparc V9
roland
parents:
23847
diff
changeset
|
5668 |
generate_stack_overflow_check(stack_bang_size); |
14626 | 5669 |
|
5670 |
// We always push rbp, so that on return to interpreter rbp, will be |
|
5671 |
// restored correctly and we can correct the stack. |
|
5672 |
push(rbp); |
|
30305
b92a97e1e9cb
8068945: Use RBP register as proper frame pointer in JIT compiled code on x86
zmajo
parents:
30299
diff
changeset
|
5673 |
// Save caller's stack pointer into RBP if the frame pointer is preserved. |
b92a97e1e9cb
8068945: Use RBP register as proper frame pointer in JIT compiled code on x86
zmajo
parents:
30299
diff
changeset
|
5674 |
if (PreserveFramePointer) { |
b92a97e1e9cb
8068945: Use RBP register as proper frame pointer in JIT compiled code on x86
zmajo
parents:
30299
diff
changeset
|
5675 |
mov(rbp, rsp); |
b92a97e1e9cb
8068945: Use RBP register as proper frame pointer in JIT compiled code on x86
zmajo
parents:
30299
diff
changeset
|
5676 |
} |
14626 | 5677 |
// Remove word for ebp |
5678 |
framesize -= wordSize; |
|
5679 |
||
5680 |
// Create frame |
|
5681 |
if (framesize) { |
|
5682 |
subptr(rsp, framesize); |
|
5683 |
} |
|
5684 |
} else { |
|
5685 |
// Create frame (force generation of a 4 byte immediate value) |
|
5686 |
subptr_imm32(rsp, framesize); |
|
5687 |
||
5688 |
// Save RBP register now. |
|
5689 |
framesize -= wordSize; |
|
5690 |
movptr(Address(rsp, framesize), rbp); |
|
30305
b92a97e1e9cb
8068945: Use RBP register as proper frame pointer in JIT compiled code on x86
zmajo
parents:
30299
diff
changeset
|
5691 |
// Save caller's stack pointer into RBP if the frame pointer is preserved. |
b92a97e1e9cb
8068945: Use RBP register as proper frame pointer in JIT compiled code on x86
zmajo
parents:
30299
diff
changeset
|
5692 |
if (PreserveFramePointer) { |
b92a97e1e9cb
8068945: Use RBP register as proper frame pointer in JIT compiled code on x86
zmajo
parents:
30299
diff
changeset
|
5693 |
movptr(rbp, rsp); |
33188
6d91a3077eca
8080650: Enable stubs to use frame pointers correctly
zmajo
parents:
33160
diff
changeset
|
5694 |
if (framesize > 0) { |
6d91a3077eca
8080650: Enable stubs to use frame pointers correctly
zmajo
parents:
33160
diff
changeset
|
5695 |
addptr(rbp, framesize); |
6d91a3077eca
8080650: Enable stubs to use frame pointers correctly
zmajo
parents:
33160
diff
changeset
|
5696 |
} |
30305
b92a97e1e9cb
8068945: Use RBP register as proper frame pointer in JIT compiled code on x86
zmajo
parents:
30299
diff
changeset
|
5697 |
} |
14626 | 5698 |
} |
5699 |
||
5700 |
if (VerifyStackAtCalls) { // Majik cookie to verify stack depth |
|
5701 |
framesize -= wordSize; |
|
5702 |
movptr(Address(rsp, framesize), (int32_t)0xbadb100d); |
|
5703 |
} |
|
5704 |
||
5705 |
#ifndef _LP64 |
|
5706 |
// If method sets FPU control word do it now |
|
5707 |
if (fp_mode_24b) { |
|
5708 |
fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24())); |
|
5709 |
} |
|
5710 |
if (UseSSE >= 2 && VerifyFPU) { |
|
5711 |
verify_FPU(0, "FPU stack must be clean on entry"); |
|
5712 |
} |
|
5713 |
#endif |
|
5714 |
||
5715 |
#ifdef ASSERT |
|
5716 |
if (VerifyStackAtCalls) { |
|
5717 |
Label L; |
|
5718 |
push(rax); |
|
5719 |
mov(rax, rsp); |
|
5720 |
andptr(rax, StackAlignmentInBytes-1); |
|
5721 |
cmpptr(rax, StackAlignmentInBytes-wordSize); |
|
5722 |
pop(rax); |
|
5723 |
jcc(Assembler::equal, L); |
|
5724 |
STOP("Stack is not properly aligned!"); |
|
5725 |
bind(L); |
|
5726 |
} |
|
5727 |
#endif |
|
5728 |
||
52142 | 5729 |
if (!is_stub) { |
5730 |
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); |
|
5731 |
bs->nmethod_entry_barrier(this); |
|
5732 |
} |
|
14626 | 5733 |
} |
5734 |
||
50534 | 5735 |
// clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM registers |
5736 |
void MacroAssembler::xmm_clear_mem(Register base, Register cnt, XMMRegister xtmp) { |
|
5737 |
// cnt - number of qwords (8-byte words). |
|
5738 |
// base - start address, qword aligned. |
|
5739 |
Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end; |
|
5740 |
if (UseAVX >= 2) { |
|
5741 |
vpxor(xtmp, xtmp, xtmp, AVX_256bit); |
|
5742 |
} else { |
|
5743 |
pxor(xtmp, xtmp); |
|
5744 |
} |
|
5745 |
jmp(L_zero_64_bytes); |
|
5746 |
||
5747 |
BIND(L_loop); |
|
5748 |
if (UseAVX >= 2) { |
|
5749 |
vmovdqu(Address(base, 0), xtmp); |
|
5750 |
vmovdqu(Address(base, 32), xtmp); |
|
5751 |
} else { |
|
5752 |
movdqu(Address(base, 0), xtmp); |
|
5753 |
movdqu(Address(base, 16), xtmp); |
|
5754 |
movdqu(Address(base, 32), xtmp); |
|
5755 |
movdqu(Address(base, 48), xtmp); |
|
5756 |
} |
|
5757 |
addptr(base, 64); |
|
5758 |
||
5759 |
BIND(L_zero_64_bytes); |
|
5760 |
subptr(cnt, 8); |
|
5761 |
jccb(Assembler::greaterEqual, L_loop); |
|
5762 |
addptr(cnt, 4); |
|
5763 |
jccb(Assembler::less, L_tail); |
|
5764 |
// Copy trailing 32 bytes |
|
5765 |
if (UseAVX >= 2) { |
|
5766 |
vmovdqu(Address(base, 0), xtmp); |
|
5767 |
} else { |
|
5768 |
movdqu(Address(base, 0), xtmp); |
|
5769 |
movdqu(Address(base, 16), xtmp); |
|
5770 |
} |
|
5771 |
addptr(base, 32); |
|
5772 |
subptr(cnt, 4); |
|
5773 |
||
5774 |
BIND(L_tail); |
|
5775 |
addptr(cnt, 4); |
|
5776 |
jccb(Assembler::lessEqual, L_end); |
|
5777 |
decrement(cnt); |
|
5778 |
||
5779 |
BIND(L_sloop); |
|
5780 |
movq(Address(base, 0), xtmp); |
|
5781 |
addptr(base, 8); |
|
5782 |
decrement(cnt); |
|
5783 |
jccb(Assembler::greaterEqual, L_sloop); |
|
5784 |
BIND(L_end); |
|
5785 |
} |
|
5786 |
||
5787 |
void MacroAssembler::clear_mem(Register base, Register cnt, Register tmp, XMMRegister xtmp, bool is_large) { |
|
15114
4074553c678b
8005522: use fast-string instructions on x86 for zeroing
kvn
parents:
14837
diff
changeset
|
5788 |
// cnt - number of qwords (8-byte words). |
4074553c678b
8005522: use fast-string instructions on x86 for zeroing
kvn
parents:
14837
diff
changeset
|
5789 |
// base - start address, qword aligned. |
36554
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5790 |
// is_large - if optimizers know cnt is larger than InitArrayShortSize |
15114
4074553c678b
8005522: use fast-string instructions on x86 for zeroing
kvn
parents:
14837
diff
changeset
|
5791 |
assert(base==rdi, "base register must be edi for rep stos"); |
4074553c678b
8005522: use fast-string instructions on x86 for zeroing
kvn
parents:
14837
diff
changeset
|
5792 |
assert(tmp==rax, "tmp register must be eax for rep stos"); |
4074553c678b
8005522: use fast-string instructions on x86 for zeroing
kvn
parents:
14837
diff
changeset
|
5793 |
assert(cnt==rcx, "cnt register must be ecx for rep stos"); |
36554
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5794 |
assert(InitArrayShortSize % BytesPerLong == 0, |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5795 |
"InitArrayShortSize should be the multiple of BytesPerLong"); |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5796 |
|
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5797 |
Label DONE; |
15114
4074553c678b
8005522: use fast-string instructions on x86 for zeroing
kvn
parents:
14837
diff
changeset
|
5798 |
|
50534 | 5799 |
if (!is_large || !UseXMMForObjInit) { |
5800 |
xorptr(tmp, tmp); |
|
5801 |
} |
|
36554
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5802 |
|
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5803 |
if (!is_large) { |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5804 |
Label LOOP, LONG; |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5805 |
cmpptr(cnt, InitArrayShortSize/BytesPerLong); |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5806 |
jccb(Assembler::greater, LONG); |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5807 |
|
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5808 |
NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5809 |
|
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5810 |
decrement(cnt); |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5811 |
jccb(Assembler::negative, DONE); // Zero length |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5812 |
|
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5813 |
// Use individual pointer-sized stores for small counts: |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5814 |
BIND(LOOP); |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5815 |
movptr(Address(base, cnt, Address::times_ptr), tmp); |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5816 |
decrement(cnt); |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5817 |
jccb(Assembler::greaterEqual, LOOP); |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5818 |
jmpb(DONE); |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5819 |
|
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5820 |
BIND(LONG); |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5821 |
} |
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5822 |
|
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5823 |
// Use longer rep-prefixed ops for non-small counts: |
15114
4074553c678b
8005522: use fast-string instructions on x86 for zeroing
kvn
parents:
14837
diff
changeset
|
5824 |
if (UseFastStosb) { |
36554
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5825 |
shlptr(cnt, 3); // convert to number of bytes |
15114
4074553c678b
8005522: use fast-string instructions on x86 for zeroing
kvn
parents:
14837
diff
changeset
|
5826 |
rep_stosb(); |
50534 | 5827 |
} else if (UseXMMForObjInit) { |
5828 |
movptr(tmp, base); |
|
5829 |
xmm_clear_mem(tmp, cnt, xtmp); |
|
15114
4074553c678b
8005522: use fast-string instructions on x86 for zeroing
kvn
parents:
14837
diff
changeset
|
5830 |
} else { |
36554
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5831 |
NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM |
15114
4074553c678b
8005522: use fast-string instructions on x86 for zeroing
kvn
parents:
14837
diff
changeset
|
5832 |
rep_stos(); |
4074553c678b
8005522: use fast-string instructions on x86 for zeroing
kvn
parents:
14837
diff
changeset
|
5833 |
} |
36554
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5834 |
|
a7eb9ee4680c
8146801: Allocating short arrays of non-constant size is slow
shade
parents:
36068
diff
changeset
|
5835 |
BIND(DONE); |
15114
4074553c678b
8005522: use fast-string instructions on x86 for zeroing
kvn
parents:
14837
diff
changeset
|
5836 |
} |
14626 | 5837 |
|
33628 | 5838 |
#ifdef COMPILER2 |
5839 |
||
14626 | 5840 |
// IndexOf for constant substrings with size >= 8 chars |
5841 |
// which don't need to be loaded through stack. |
|
5842 |
void MacroAssembler::string_indexofC8(Register str1, Register str2, |
|
5843 |
Register cnt1, Register cnt2, |
|
5844 |
int int_cnt2, Register result, |
|
33628 | 5845 |
XMMRegister vec, Register tmp, |
5846 |
int ae) { |
|
14626 | 5847 |
ShortBranchVerifier sbv(this); |
34207
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
5848 |
assert(UseSSE42Intrinsics, "SSE4.2 intrinsics are required"); |
33628 | 5849 |
assert(ae != StrIntrinsicNode::LU, "Invalid encoding"); |
5850 |
||
5851 |
// This method uses the pcmpestri instruction with bound registers |
|
14626 | 5852 |
// inputs: |
5853 |
// xmm - substring |
|
5854 |
// rax - substring length (elements count) |
|
5855 |
// mem - scanned string |
|
5856 |
// rdx - string length (elements count) |
|
5857 |
// 0xd - mode: 1100 (substring search) + 01 (unsigned shorts) |
|
33628 | 5858 |
// 0xc - mode: 1100 (substring search) + 00 (unsigned bytes) |
14626 | 5859 |
// outputs: |
5860 |
// rcx - matched index in string |
|
5861 |
assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri"); |
|
33628 | 5862 |
int mode = (ae == StrIntrinsicNode::LL) ? 0x0c : 0x0d; // bytes or shorts |
5863 |
int stride = (ae == StrIntrinsicNode::LL) ? 16 : 8; //UU, UL -> 8 |
|
5864 |
Address::ScaleFactor scale1 = (ae == StrIntrinsicNode::LL) ? Address::times_1 : Address::times_2; |
|
5865 |
Address::ScaleFactor scale2 = (ae == StrIntrinsicNode::UL) ? Address::times_1 : scale1; |
|
14626 | 5866 |
|
5867 |
Label RELOAD_SUBSTR, SCAN_TO_SUBSTR, SCAN_SUBSTR, |
|
5868 |
RET_FOUND, RET_NOT_FOUND, EXIT, FOUND_SUBSTR, |
|
5869 |
MATCH_SUBSTR_HEAD, RELOAD_STR, FOUND_CANDIDATE; |
|
5870 |
||
5871 |
// Note, inline_string_indexOf() generates checks: |
|
5872 |
// if (substr.count > string.count) return -1; |
|
5873 |
// if (substr.count == 0) return 0; |
|
33628 | 5874 |
assert(int_cnt2 >= stride, "this code is used only for cnt2 >= 8 chars"); |
14626 | 5875 |
|
5876 |
// Load substring. |
|
33628 | 5877 |
if (ae == StrIntrinsicNode::UL) { |
5878 |
pmovzxbw(vec, Address(str2, 0)); |
|
5879 |
} else { |
|
5880 |
movdqu(vec, Address(str2, 0)); |
|
5881 |
} |
|
14626 | 5882 |
movl(cnt2, int_cnt2); |
5883 |
movptr(result, str1); // string addr |
|
5884 |
||
33628 | 5885 |
if (int_cnt2 > stride) { |
14626 | 5886 |
jmpb(SCAN_TO_SUBSTR); |
5887 |
||
5888 |
// Reload substr for rescan, this code |
|
5889 |
// is executed only for large substrings (> 8 chars) |
|
5890 |
bind(RELOAD_SUBSTR); |
|
33628 | 5891 |
if (ae == StrIntrinsicNode::UL) { |
5892 |
pmovzxbw(vec, Address(str2, 0)); |
|
5893 |
} else { |
|
5894 |
movdqu(vec, Address(str2, 0)); |
|
5895 |
} |
|
14626 | 5896 |
negptr(cnt2); // Jumped here with negative cnt2, convert to positive |
5897 |
||
5898 |
bind(RELOAD_STR); |
|
5899 |
// We came here after the beginning of the substring was |
|
5900 |
// matched but the rest of it was not so we need to search |
|
5901 |
// again. Start from the next element after the previous match. |
|
5902 |
||
5903 |
// cnt2 is number of substring reminding elements and |
|
5904 |
// cnt1 is number of string reminding elements when cmp failed. |
|
5905 |
// Restored cnt1 = cnt1 - cnt2 + int_cnt2 |
|
5906 |
subl(cnt1, cnt2); |
|
5907 |
addl(cnt1, int_cnt2); |
|
5908 |
movl(cnt2, int_cnt2); // Now restore cnt2 |
|
5909 |
||
5910 |
decrementl(cnt1); // Shift to next element |
|
5911 |
cmpl(cnt1, cnt2); |
|
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
5912 |
jcc(Assembler::negative, RET_NOT_FOUND); // Left less then substring |
14626 | 5913 |
|
33628 | 5914 |
addptr(result, (1<<scale1)); |
14626 | 5915 |
|
5916 |
} // (int_cnt2 > 8) |
|
5917 |
||
5918 |
// Scan string for start of substr in 16-byte vectors |
|
5919 |
bind(SCAN_TO_SUBSTR); |
|
33628 | 5920 |
pcmpestri(vec, Address(result, 0), mode); |
14626 | 5921 |
jccb(Assembler::below, FOUND_CANDIDATE); // CF == 1 |
33628 | 5922 |
subl(cnt1, stride); |
14626 | 5923 |
jccb(Assembler::lessEqual, RET_NOT_FOUND); // Scanned full string |
5924 |
cmpl(cnt1, cnt2); |
|
5925 |
jccb(Assembler::negative, RET_NOT_FOUND); // Left less then substring |
|
5926 |
addptr(result, 16); |
|
5927 |
jmpb(SCAN_TO_SUBSTR); |
|
5928 |
||
5929 |
// Found a potential substr |
|
5930 |
bind(FOUND_CANDIDATE); |
|
5931 |
// Matched whole vector if first element matched (tmp(rcx) == 0). |
|
33628 | 5932 |
if (int_cnt2 == stride) { |
14626 | 5933 |
jccb(Assembler::overflow, RET_FOUND); // OF == 1 |
5934 |
} else { // int_cnt2 > 8 |
|
5935 |
jccb(Assembler::overflow, FOUND_SUBSTR); |
|
5936 |
} |
|
5937 |
// After pcmpestri tmp(rcx) contains matched element index |
|
5938 |
// Compute start addr of substr |
|
33628 | 5939 |
lea(result, Address(result, tmp, scale1)); |
14626 | 5940 |
|
5941 |
// Make sure string is still long enough |
|
5942 |
subl(cnt1, tmp); |
|
5943 |
cmpl(cnt1, cnt2); |
|
33628 | 5944 |
if (int_cnt2 == stride) { |
14626 | 5945 |
jccb(Assembler::greaterEqual, SCAN_TO_SUBSTR); |
5946 |
} else { // int_cnt2 > 8 |
|
5947 |
jccb(Assembler::greaterEqual, MATCH_SUBSTR_HEAD); |
|
5948 |
} |
|
5949 |
// Left less then substring. |
|
5950 |
||
5951 |
bind(RET_NOT_FOUND); |
|
5952 |
movl(result, -1); |
|
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
5953 |
jmp(EXIT); |
14626 | 5954 |
|
33628 | 5955 |
if (int_cnt2 > stride) { |
14626 | 5956 |
// This code is optimized for the case when whole substring |
5957 |
// is matched if its head is matched. |
|
5958 |
bind(MATCH_SUBSTR_HEAD); |
|
33628 | 5959 |
pcmpestri(vec, Address(result, 0), mode); |
14626 | 5960 |
// Reload only string if does not match |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
5961 |
jcc(Assembler::noOverflow, RELOAD_STR); // OF == 0 |
14626 | 5962 |
|
5963 |
Label CONT_SCAN_SUBSTR; |
|
5964 |
// Compare the rest of substring (> 8 chars). |
|
5965 |
bind(FOUND_SUBSTR); |
|
5966 |
// First 8 chars are already matched. |
|
5967 |
negptr(cnt2); |
|
33628 | 5968 |
addptr(cnt2, stride); |
14626 | 5969 |
|
5970 |
bind(SCAN_SUBSTR); |
|
33628 | 5971 |
subl(cnt1, stride); |
5972 |
cmpl(cnt2, -stride); // Do not read beyond substring |
|
14626 | 5973 |
jccb(Assembler::lessEqual, CONT_SCAN_SUBSTR); |
5974 |
// Back-up strings to avoid reading beyond substring: |
|
5975 |
// cnt1 = cnt1 - cnt2 + 8 |
|
5976 |
addl(cnt1, cnt2); // cnt2 is negative |
|
33628 | 5977 |
addl(cnt1, stride); |
5978 |
movl(cnt2, stride); negptr(cnt2); |
|
14626 | 5979 |
bind(CONT_SCAN_SUBSTR); |
5980 |
if (int_cnt2 < (int)G) { |
|
33628 | 5981 |
int tail_off1 = int_cnt2<<scale1; |
5982 |
int tail_off2 = int_cnt2<<scale2; |
|
5983 |
if (ae == StrIntrinsicNode::UL) { |
|
5984 |
pmovzxbw(vec, Address(str2, cnt2, scale2, tail_off2)); |
|
5985 |
} else { |
|
5986 |
movdqu(vec, Address(str2, cnt2, scale2, tail_off2)); |
|
5987 |
} |
|
5988 |
pcmpestri(vec, Address(result, cnt2, scale1, tail_off1), mode); |
|
14626 | 5989 |
} else { |
5990 |
// calculate index in register to avoid integer overflow (int_cnt2*2) |
|
5991 |
movl(tmp, int_cnt2); |
|
5992 |
addptr(tmp, cnt2); |
|
33628 | 5993 |
if (ae == StrIntrinsicNode::UL) { |
5994 |
pmovzxbw(vec, Address(str2, tmp, scale2, 0)); |
|
5995 |
} else { |
|
5996 |
movdqu(vec, Address(str2, tmp, scale2, 0)); |
|
5997 |
} |
|
5998 |
pcmpestri(vec, Address(result, tmp, scale1, 0), mode); |
|
14626 | 5999 |
} |
6000 |
// Need to reload strings pointers if not matched whole vector |
|
6001 |
jcc(Assembler::noOverflow, RELOAD_SUBSTR); // OF == 0 |
|
33628 | 6002 |
addptr(cnt2, stride); |
14626 | 6003 |
jcc(Assembler::negative, SCAN_SUBSTR); |
6004 |
// Fall through if found full substring |
|
6005 |
||
6006 |
} // (int_cnt2 > 8) |
|
6007 |
||
6008 |
bind(RET_FOUND); |
|
6009 |
// Found result if we matched full small substring. |
|
6010 |
// Compute substr offset |
|
6011 |
subptr(result, str1); |
|
33628 | 6012 |
if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) { |
6013 |
shrl(result, 1); // index |
|
6014 |
} |
|
14626 | 6015 |
bind(EXIT); |
6016 |
||
6017 |
} // string_indexofC8 |
|
6018 |
||
6019 |
// Small strings are loaded through stack if they cross page boundary. |
|
6020 |
void MacroAssembler::string_indexof(Register str1, Register str2, |
|
6021 |
Register cnt1, Register cnt2, |
|
6022 |
int int_cnt2, Register result, |
|
33628 | 6023 |
XMMRegister vec, Register tmp, |
6024 |
int ae) { |
|
14626 | 6025 |
ShortBranchVerifier sbv(this); |
34207
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
6026 |
assert(UseSSE42Intrinsics, "SSE4.2 intrinsics are required"); |
33628 | 6027 |
assert(ae != StrIntrinsicNode::LU, "Invalid encoding"); |
6028 |
||
14626 | 6029 |
// |
6030 |
// int_cnt2 is length of small (< 8 chars) constant substring |
|
6031 |
// or (-1) for non constant substring in which case its length |
|
6032 |
// is in cnt2 register. |
|
6033 |
// |
|
6034 |
// Note, inline_string_indexOf() generates checks: |
|
6035 |
// if (substr.count > string.count) return -1; |
|
6036 |
// if (substr.count == 0) return 0; |
|
6037 |
// |
|
33628 | 6038 |
int stride = (ae == StrIntrinsicNode::LL) ? 16 : 8; //UU, UL -> 8 |
6039 |
assert(int_cnt2 == -1 || (0 < int_cnt2 && int_cnt2 < stride), "should be != 0"); |
|
6040 |
// This method uses the pcmpestri instruction with bound registers |
|
14626 | 6041 |
// inputs: |
6042 |
// xmm - substring |
|
6043 |
// rax - substring length (elements count) |
|
6044 |
// mem - scanned string |
|
6045 |
// rdx - string length (elements count) |
|
6046 |
// 0xd - mode: 1100 (substring search) + 01 (unsigned shorts) |
|
33628 | 6047 |
// 0xc - mode: 1100 (substring search) + 00 (unsigned bytes) |
14626 | 6048 |
// outputs: |
6049 |
// rcx - matched index in string |
|
6050 |
assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri"); |
|
33628 | 6051 |
int mode = (ae == StrIntrinsicNode::LL) ? 0x0c : 0x0d; // bytes or shorts |
6052 |
Address::ScaleFactor scale1 = (ae == StrIntrinsicNode::LL) ? Address::times_1 : Address::times_2; |
|
6053 |
Address::ScaleFactor scale2 = (ae == StrIntrinsicNode::UL) ? Address::times_1 : scale1; |
|
14626 | 6054 |
|
6055 |
Label RELOAD_SUBSTR, SCAN_TO_SUBSTR, SCAN_SUBSTR, ADJUST_STR, |
|
6056 |
RET_FOUND, RET_NOT_FOUND, CLEANUP, FOUND_SUBSTR, |
|
6057 |
FOUND_CANDIDATE; |
|
6058 |
||
6059 |
{ //======================================================== |
|
6060 |
// We don't know where these strings are located |
|
6061 |
// and we can't read beyond them. Load them through stack. |
|
6062 |
Label BIG_STRINGS, CHECK_STR, COPY_SUBSTR, COPY_STR; |
|
6063 |
||
6064 |
movptr(tmp, rsp); // save old SP |
|
6065 |
||
6066 |
if (int_cnt2 > 0) { // small (< 8 chars) constant substring |
|
33628 | 6067 |
if (int_cnt2 == (1>>scale2)) { // One byte |
6068 |
assert((ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL), "Only possible for latin1 encoding"); |
|
6069 |
load_unsigned_byte(result, Address(str2, 0)); |
|
6070 |
movdl(vec, result); // move 32 bits |
|
6071 |
} else if (ae == StrIntrinsicNode::LL && int_cnt2 == 3) { // Three bytes |
|
6072 |
// Not enough header space in 32-bit VM: 12+3 = 15. |
|
6073 |
movl(result, Address(str2, -1)); |
|
6074 |
shrl(result, 8); |
|
6075 |
movdl(vec, result); // move 32 bits |
|
6076 |
} else if (ae != StrIntrinsicNode::UL && int_cnt2 == (2>>scale2)) { // One char |
|
14626 | 6077 |
load_unsigned_short(result, Address(str2, 0)); |
6078 |
movdl(vec, result); // move 32 bits |
|
33628 | 6079 |
} else if (ae != StrIntrinsicNode::UL && int_cnt2 == (4>>scale2)) { // Two chars |
14626 | 6080 |
movdl(vec, Address(str2, 0)); // move 32 bits |
33628 | 6081 |
} else if (ae != StrIntrinsicNode::UL && int_cnt2 == (8>>scale2)) { // Four chars |
14626 | 6082 |
movq(vec, Address(str2, 0)); // move 64 bits |
33628 | 6083 |
} else { // cnt2 = { 3, 5, 6, 7 } || (ae == StrIntrinsicNode::UL && cnt2 ={2, ..., 7}) |
14626 | 6084 |
// Array header size is 12 bytes in 32-bit VM |
6085 |
// + 6 bytes for 3 chars == 18 bytes, |
|
6086 |
// enough space to load vec and shift. |
|
6087 |
assert(HeapWordSize*TypeArrayKlass::header_size() >= 12,"sanity"); |
|
33628 | 6088 |
if (ae == StrIntrinsicNode::UL) { |
6089 |
int tail_off = int_cnt2-8; |
|
6090 |
pmovzxbw(vec, Address(str2, tail_off)); |
|
6091 |
psrldq(vec, -2*tail_off); |
|
6092 |
} |
|
6093 |
else { |
|
6094 |
int tail_off = int_cnt2*(1<<scale2); |
|
6095 |
movdqu(vec, Address(str2, tail_off-16)); |
|
6096 |
psrldq(vec, 16-tail_off); |
|
6097 |
} |
|
14626 | 6098 |
} |
6099 |
} else { // not constant substring |
|
33628 | 6100 |
cmpl(cnt2, stride); |
14626 | 6101 |
jccb(Assembler::aboveEqual, BIG_STRINGS); // Both strings are big enough |
6102 |
||
6103 |
// We can read beyond string if srt+16 does not cross page boundary |
|
6104 |
// since heaps are aligned and mapped by pages. |
|
6105 |
assert(os::vm_page_size() < (int)G, "default page should be small"); |
|
6106 |
movl(result, str2); // We need only low 32 bits |
|
6107 |
andl(result, (os::vm_page_size()-1)); |
|
6108 |
cmpl(result, (os::vm_page_size()-16)); |
|
6109 |
jccb(Assembler::belowEqual, CHECK_STR); |
|
6110 |
||
6111 |
// Move small strings to stack to allow load 16 bytes into vec. |
|
6112 |
subptr(rsp, 16); |
|
33628 | 6113 |
int stk_offset = wordSize-(1<<scale2); |
14626 | 6114 |
push(cnt2); |
6115 |
||
6116 |
bind(COPY_SUBSTR); |
|
33628 | 6117 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL) { |
6118 |
load_unsigned_byte(result, Address(str2, cnt2, scale2, -1)); |
|
6119 |
movb(Address(rsp, cnt2, scale2, stk_offset), result); |
|
6120 |
} else if (ae == StrIntrinsicNode::UU) { |
|
6121 |
load_unsigned_short(result, Address(str2, cnt2, scale2, -2)); |
|
6122 |
movw(Address(rsp, cnt2, scale2, stk_offset), result); |
|
6123 |
} |
|
14626 | 6124 |
decrement(cnt2); |
6125 |
jccb(Assembler::notZero, COPY_SUBSTR); |
|
6126 |
||
6127 |
pop(cnt2); |
|
6128 |
movptr(str2, rsp); // New substring address |
|
6129 |
} // non constant |
|
6130 |
||
6131 |
bind(CHECK_STR); |
|
33628 | 6132 |
cmpl(cnt1, stride); |
14626 | 6133 |
jccb(Assembler::aboveEqual, BIG_STRINGS); |
6134 |
||
6135 |
// Check cross page boundary. |
|
6136 |
movl(result, str1); // We need only low 32 bits |
|
6137 |
andl(result, (os::vm_page_size()-1)); |
|
6138 |
cmpl(result, (os::vm_page_size()-16)); |
|
6139 |
jccb(Assembler::belowEqual, BIG_STRINGS); |
|
6140 |
||
6141 |
subptr(rsp, 16); |
|
33628 | 6142 |
int stk_offset = -(1<<scale1); |
14626 | 6143 |
if (int_cnt2 < 0) { // not constant |
6144 |
push(cnt2); |
|
6145 |
stk_offset += wordSize; |
|
6146 |
} |
|
6147 |
movl(cnt2, cnt1); |
|
6148 |
||
6149 |
bind(COPY_STR); |
|
33628 | 6150 |
if (ae == StrIntrinsicNode::LL) { |
6151 |
load_unsigned_byte(result, Address(str1, cnt2, scale1, -1)); |
|
6152 |
movb(Address(rsp, cnt2, scale1, stk_offset), result); |
|
6153 |
} else { |
|
6154 |
load_unsigned_short(result, Address(str1, cnt2, scale1, -2)); |
|
6155 |
movw(Address(rsp, cnt2, scale1, stk_offset), result); |
|
6156 |
} |
|
14626 | 6157 |
decrement(cnt2); |
6158 |
jccb(Assembler::notZero, COPY_STR); |
|
6159 |
||
6160 |
if (int_cnt2 < 0) { // not constant |
|
6161 |
pop(cnt2); |
|
6162 |
} |
|
6163 |
movptr(str1, rsp); // New string address |
|
6164 |
||
6165 |
bind(BIG_STRINGS); |
|
6166 |
// Load substring. |
|
6167 |
if (int_cnt2 < 0) { // -1 |
|
33628 | 6168 |
if (ae == StrIntrinsicNode::UL) { |
6169 |
pmovzxbw(vec, Address(str2, 0)); |
|
6170 |
} else { |
|
6171 |
movdqu(vec, Address(str2, 0)); |
|
6172 |
} |
|
14626 | 6173 |
push(cnt2); // substr count |
6174 |
push(str2); // substr addr |
|
6175 |
push(str1); // string addr |
|
6176 |
} else { |
|
6177 |
// Small (< 8 chars) constant substrings are loaded already. |
|
6178 |
movl(cnt2, int_cnt2); |
|
6179 |
} |
|
6180 |
push(tmp); // original SP |
|
6181 |
||
6182 |
} // Finished loading |
|
6183 |
||
6184 |
//======================================================== |
|
6185 |
// Start search |
|
6186 |
// |
|
6187 |
||
6188 |
movptr(result, str1); // string addr |
|
6189 |
||
6190 |
if (int_cnt2 < 0) { // Only for non constant substring |
|
6191 |
jmpb(SCAN_TO_SUBSTR); |
|
6192 |
||
6193 |
// SP saved at sp+0 |
|
6194 |
// String saved at sp+1*wordSize |
|
6195 |
// Substr saved at sp+2*wordSize |
|
6196 |
// Substr count saved at sp+3*wordSize |
|
6197 |
||
6198 |
// Reload substr for rescan, this code |
|
6199 |
// is executed only for large substrings (> 8 chars) |
|
6200 |
bind(RELOAD_SUBSTR); |
|
6201 |
movptr(str2, Address(rsp, 2*wordSize)); |
|
6202 |
movl(cnt2, Address(rsp, 3*wordSize)); |
|
33628 | 6203 |
if (ae == StrIntrinsicNode::UL) { |
6204 |
pmovzxbw(vec, Address(str2, 0)); |
|
6205 |
} else { |
|
6206 |
movdqu(vec, Address(str2, 0)); |
|
6207 |
} |
|
14626 | 6208 |
// We came here after the beginning of the substring was |
6209 |
// matched but the rest of it was not so we need to search |
|
6210 |
// again. Start from the next element after the previous match. |
|
6211 |
subptr(str1, result); // Restore counter |
|
33628 | 6212 |
if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) { |
6213 |
shrl(str1, 1); |
|
6214 |
} |
|
14626 | 6215 |
addl(cnt1, str1); |
6216 |
decrementl(cnt1); // Shift to next element |
|
6217 |
cmpl(cnt1, cnt2); |
|
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
6218 |
jcc(Assembler::negative, RET_NOT_FOUND); // Left less then substring |
14626 | 6219 |
|
33628 | 6220 |
addptr(result, (1<<scale1)); |
14626 | 6221 |
} // non constant |
6222 |
||
6223 |
// Scan string for start of substr in 16-byte vectors |
|
6224 |
bind(SCAN_TO_SUBSTR); |
|
6225 |
assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri"); |
|
33628 | 6226 |
pcmpestri(vec, Address(result, 0), mode); |
14626 | 6227 |
jccb(Assembler::below, FOUND_CANDIDATE); // CF == 1 |
33628 | 6228 |
subl(cnt1, stride); |
14626 | 6229 |
jccb(Assembler::lessEqual, RET_NOT_FOUND); // Scanned full string |
6230 |
cmpl(cnt1, cnt2); |
|
6231 |
jccb(Assembler::negative, RET_NOT_FOUND); // Left less then substring |
|
6232 |
addptr(result, 16); |
|
6233 |
||
6234 |
bind(ADJUST_STR); |
|
33628 | 6235 |
cmpl(cnt1, stride); // Do not read beyond string |
14626 | 6236 |
jccb(Assembler::greaterEqual, SCAN_TO_SUBSTR); |
6237 |
// Back-up string to avoid reading beyond string. |
|
33628 | 6238 |
lea(result, Address(result, cnt1, scale1, -16)); |
6239 |
movl(cnt1, stride); |
|
14626 | 6240 |
jmpb(SCAN_TO_SUBSTR); |
6241 |
||
6242 |
// Found a potential substr |
|
6243 |
bind(FOUND_CANDIDATE); |
|
6244 |
// After pcmpestri tmp(rcx) contains matched element index |
|
6245 |
||
6246 |
// Make sure string is still long enough |
|
6247 |
subl(cnt1, tmp); |
|
6248 |
cmpl(cnt1, cnt2); |
|
6249 |
jccb(Assembler::greaterEqual, FOUND_SUBSTR); |
|
6250 |
// Left less then substring. |
|
6251 |
||
6252 |
bind(RET_NOT_FOUND); |
|
6253 |
movl(result, -1); |
|
51633
21154cb84d2a
8209594: guarantee(this->is8bit(imm8)) failed: Short forward jump exceeds 8-bit offset
kvn
parents:
51464
diff
changeset
|
6254 |
jmp(CLEANUP); |
14626 | 6255 |
|
6256 |
bind(FOUND_SUBSTR); |
|
6257 |
// Compute start addr of substr |
|
33628 | 6258 |
lea(result, Address(result, tmp, scale1)); |
14626 | 6259 |
if (int_cnt2 > 0) { // Constant substring |
6260 |
// Repeat search for small substring (< 8 chars) |
|
6261 |
// from new point without reloading substring. |
|
6262 |
// Have to check that we don't read beyond string. |
|
33628 | 6263 |
cmpl(tmp, stride-int_cnt2); |
14626 | 6264 |
jccb(Assembler::greater, ADJUST_STR); |
6265 |
// Fall through if matched whole substring. |
|
6266 |
} else { // non constant |
|
6267 |
assert(int_cnt2 == -1, "should be != 0"); |
|
6268 |
||
6269 |
addl(tmp, cnt2); |
|
6270 |
// Found result if we matched whole substring. |
|
33628 | 6271 |
cmpl(tmp, stride); |
51633
21154cb84d2a
8209594: guarantee(this->is8bit(imm8)) failed: Short forward jump exceeds 8-bit offset
kvn
parents:
51464
diff
changeset
|
6272 |
jcc(Assembler::lessEqual, RET_FOUND); |
14626 | 6273 |
|
6274 |
// Repeat search for small substring (<= 8 chars) |
|
6275 |
// from new point 'str1' without reloading substring. |
|
33628 | 6276 |
cmpl(cnt2, stride); |
14626 | 6277 |
// Have to check that we don't read beyond string. |
6278 |
jccb(Assembler::lessEqual, ADJUST_STR); |
|
6279 |
||
6280 |
Label CHECK_NEXT, CONT_SCAN_SUBSTR, RET_FOUND_LONG; |
|
6281 |
// Compare the rest of substring (> 8 chars). |
|
6282 |
movptr(str1, result); |
|
6283 |
||
6284 |
cmpl(tmp, cnt2); |
|
6285 |
// First 8 chars are already matched. |
|
6286 |
jccb(Assembler::equal, CHECK_NEXT); |
|
6287 |
||
6288 |
bind(SCAN_SUBSTR); |
|
33628 | 6289 |
pcmpestri(vec, Address(str1, 0), mode); |
14626 | 6290 |
// Need to reload strings pointers if not matched whole vector |
6291 |
jcc(Assembler::noOverflow, RELOAD_SUBSTR); // OF == 0 |
|
6292 |
||
6293 |
bind(CHECK_NEXT); |
|
33628 | 6294 |
subl(cnt2, stride); |
14626 | 6295 |
jccb(Assembler::lessEqual, RET_FOUND_LONG); // Found full substring |
6296 |
addptr(str1, 16); |
|
33628 | 6297 |
if (ae == StrIntrinsicNode::UL) { |
6298 |
addptr(str2, 8); |
|
6299 |
} else { |
|
6300 |
addptr(str2, 16); |
|
6301 |
} |
|
6302 |
subl(cnt1, stride); |
|
6303 |
cmpl(cnt2, stride); // Do not read beyond substring |
|
14626 | 6304 |
jccb(Assembler::greaterEqual, CONT_SCAN_SUBSTR); |
6305 |
// Back-up strings to avoid reading beyond substring. |
|
33628 | 6306 |
|
6307 |
if (ae == StrIntrinsicNode::UL) { |
|
6308 |
lea(str2, Address(str2, cnt2, scale2, -8)); |
|
6309 |
lea(str1, Address(str1, cnt2, scale1, -16)); |
|
6310 |
} else { |
|
6311 |
lea(str2, Address(str2, cnt2, scale2, -16)); |
|
6312 |
lea(str1, Address(str1, cnt2, scale1, -16)); |
|
6313 |
} |
|
14626 | 6314 |
subl(cnt1, cnt2); |
33628 | 6315 |
movl(cnt2, stride); |
6316 |
addl(cnt1, stride); |
|
14626 | 6317 |
bind(CONT_SCAN_SUBSTR); |
33628 | 6318 |
if (ae == StrIntrinsicNode::UL) { |
6319 |
pmovzxbw(vec, Address(str2, 0)); |
|
6320 |
} else { |
|
6321 |
movdqu(vec, Address(str2, 0)); |
|
6322 |
} |
|
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
6323 |
jmp(SCAN_SUBSTR); |
14626 | 6324 |
|
6325 |
bind(RET_FOUND_LONG); |
|
6326 |
movptr(str1, Address(rsp, wordSize)); |
|
6327 |
} // non constant |
|
6328 |
||
6329 |
bind(RET_FOUND); |
|
6330 |
// Compute substr offset |
|
6331 |
subptr(result, str1); |
|
33628 | 6332 |
if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) { |
6333 |
shrl(result, 1); // index |
|
6334 |
} |
|
14626 | 6335 |
bind(CLEANUP); |
6336 |
pop(rsp); // restore SP |
|
6337 |
||
6338 |
} // string_indexof |
|
6339 |
||
33628 | 6340 |
void MacroAssembler::string_indexof_char(Register str1, Register cnt1, Register ch, Register result, |
6341 |
XMMRegister vec1, XMMRegister vec2, XMMRegister vec3, Register tmp) { |
|
6342 |
ShortBranchVerifier sbv(this); |
|
34207
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
6343 |
assert(UseSSE42Intrinsics, "SSE4.2 intrinsics are required"); |
33628 | 6344 |
|
6345 |
int stride = 8; |
|
6346 |
||
6347 |
Label FOUND_CHAR, SCAN_TO_CHAR, SCAN_TO_CHAR_LOOP, |
|
6348 |
SCAN_TO_8_CHAR, SCAN_TO_8_CHAR_LOOP, SCAN_TO_16_CHAR_LOOP, |
|
6349 |
RET_NOT_FOUND, SCAN_TO_8_CHAR_INIT, |
|
6350 |
FOUND_SEQ_CHAR, DONE_LABEL; |
|
6351 |
||
6352 |
movptr(result, str1); |
|
6353 |
if (UseAVX >= 2) { |
|
6354 |
cmpl(cnt1, stride); |
|
58625 | 6355 |
jcc(Assembler::less, SCAN_TO_CHAR); |
33628 | 6356 |
cmpl(cnt1, 2*stride); |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
6357 |
jcc(Assembler::less, SCAN_TO_8_CHAR_INIT); |
33628 | 6358 |
movdl(vec1, ch); |
51857 | 6359 |
vpbroadcastw(vec1, vec1, Assembler::AVX_256bit); |
33628 | 6360 |
vpxor(vec2, vec2); |
6361 |
movl(tmp, cnt1); |
|
6362 |
andl(tmp, 0xFFFFFFF0); //vector count (in chars) |
|
6363 |
andl(cnt1,0x0000000F); //tail count (in chars) |
|
6364 |
||
6365 |
bind(SCAN_TO_16_CHAR_LOOP); |
|
6366 |
vmovdqu(vec3, Address(result, 0)); |
|
34162 | 6367 |
vpcmpeqw(vec3, vec3, vec1, 1); |
33628 | 6368 |
vptest(vec2, vec3); |
6369 |
jcc(Assembler::carryClear, FOUND_CHAR); |
|
6370 |
addptr(result, 32); |
|
6371 |
subl(tmp, 2*stride); |
|
51633
21154cb84d2a
8209594: guarantee(this->is8bit(imm8)) failed: Short forward jump exceeds 8-bit offset
kvn
parents:
51464
diff
changeset
|
6372 |
jcc(Assembler::notZero, SCAN_TO_16_CHAR_LOOP); |
33628 | 6373 |
jmp(SCAN_TO_8_CHAR); |
6374 |
bind(SCAN_TO_8_CHAR_INIT); |
|
6375 |
movdl(vec1, ch); |
|
6376 |
pshuflw(vec1, vec1, 0x00); |
|
6377 |
pshufd(vec1, vec1, 0); |
|
6378 |
pxor(vec2, vec2); |
|
6379 |
} |
|
34207
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
6380 |
bind(SCAN_TO_8_CHAR); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
6381 |
cmpl(cnt1, stride); |
58625 | 6382 |
jcc(Assembler::less, SCAN_TO_CHAR); |
6383 |
if (UseAVX < 2) { |
|
34207
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
6384 |
movdl(vec1, ch); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
6385 |
pshuflw(vec1, vec1, 0x00); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
6386 |
pshufd(vec1, vec1, 0); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
6387 |
pxor(vec2, vec2); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
6388 |
} |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
6389 |
movl(tmp, cnt1); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
6390 |
andl(tmp, 0xFFFFFFF8); //vector count (in chars) |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
6391 |
andl(cnt1,0x00000007); //tail count (in chars) |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
6392 |
|
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
6393 |
bind(SCAN_TO_8_CHAR_LOOP); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
6394 |
movdqu(vec3, Address(result, 0)); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
6395 |
pcmpeqw(vec3, vec1); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
6396 |
ptest(vec2, vec3); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
6397 |
jcc(Assembler::carryClear, FOUND_CHAR); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
6398 |
addptr(result, 16); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
6399 |
subl(tmp, stride); |
51633
21154cb84d2a
8209594: guarantee(this->is8bit(imm8)) failed: Short forward jump exceeds 8-bit offset
kvn
parents:
51464
diff
changeset
|
6400 |
jcc(Assembler::notZero, SCAN_TO_8_CHAR_LOOP); |
33628 | 6401 |
bind(SCAN_TO_CHAR); |
6402 |
testl(cnt1, cnt1); |
|
6403 |
jcc(Assembler::zero, RET_NOT_FOUND); |
|
6404 |
bind(SCAN_TO_CHAR_LOOP); |
|
6405 |
load_unsigned_short(tmp, Address(result, 0)); |
|
6406 |
cmpl(ch, tmp); |
|
6407 |
jccb(Assembler::equal, FOUND_SEQ_CHAR); |
|
6408 |
addptr(result, 2); |
|
6409 |
subl(cnt1, 1); |
|
6410 |
jccb(Assembler::zero, RET_NOT_FOUND); |
|
6411 |
jmp(SCAN_TO_CHAR_LOOP); |
|
6412 |
||
6413 |
bind(RET_NOT_FOUND); |
|
6414 |
movl(result, -1); |
|
6415 |
jmpb(DONE_LABEL); |
|
6416 |
||
34207
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
6417 |
bind(FOUND_CHAR); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
6418 |
if (UseAVX >= 2) { |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
6419 |
vpmovmskb(tmp, vec3); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
6420 |
} else { |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
6421 |
pmovmskb(tmp, vec3); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
6422 |
} |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
6423 |
bsfl(ch, tmp); |
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
6424 |
addl(result, ch); |
33628 | 6425 |
|
6426 |
bind(FOUND_SEQ_CHAR); |
|
6427 |
subptr(result, str1); |
|
6428 |
shrl(result, 1); |
|
6429 |
||
6430 |
bind(DONE_LABEL); |
|
6431 |
} // string_indexof_char |
|
6432 |
||
6433 |
// helper function for string_compare |
|
6434 |
void MacroAssembler::load_next_elements(Register elem1, Register elem2, Register str1, Register str2, |
|
6435 |
Address::ScaleFactor scale, Address::ScaleFactor scale1, |
|
6436 |
Address::ScaleFactor scale2, Register index, int ae) { |
|
6437 |
if (ae == StrIntrinsicNode::LL) { |
|
6438 |
load_unsigned_byte(elem1, Address(str1, index, scale, 0)); |
|
6439 |
load_unsigned_byte(elem2, Address(str2, index, scale, 0)); |
|
6440 |
} else if (ae == StrIntrinsicNode::UU) { |
|
6441 |
load_unsigned_short(elem1, Address(str1, index, scale, 0)); |
|
6442 |
load_unsigned_short(elem2, Address(str2, index, scale, 0)); |
|
6443 |
} else { |
|
6444 |
load_unsigned_byte(elem1, Address(str1, index, scale1, 0)); |
|
6445 |
load_unsigned_short(elem2, Address(str2, index, scale2, 0)); |
|
6446 |
} |
|
6447 |
} |
|
6448 |
||
6449 |
// Compare strings, used for char[] and byte[]. |
|
14626 | 6450 |
void MacroAssembler::string_compare(Register str1, Register str2, |
6451 |
Register cnt1, Register cnt2, Register result, |
|
33628 | 6452 |
XMMRegister vec1, int ae) { |
14626 | 6453 |
ShortBranchVerifier sbv(this); |
6454 |
Label LENGTH_DIFF_LABEL, POP_LABEL, DONE_LABEL, WHILE_HEAD_LABEL; |
|
35113 | 6455 |
Label COMPARE_WIDE_VECTORS_LOOP_FAILED; // used only _LP64 && AVX3 |
33628 | 6456 |
int stride, stride2, adr_stride, adr_stride1, adr_stride2; |
35113 | 6457 |
int stride2x2 = 0x40; |
36061
baa556050d22
8145700: Uninitialised variable in macroAssembler_x86.cpp:7038
thartmann
parents:
35548
diff
changeset
|
6458 |
Address::ScaleFactor scale = Address::no_scale; |
baa556050d22
8145700: Uninitialised variable in macroAssembler_x86.cpp:7038
thartmann
parents:
35548
diff
changeset
|
6459 |
Address::ScaleFactor scale1 = Address::no_scale; |
baa556050d22
8145700: Uninitialised variable in macroAssembler_x86.cpp:7038
thartmann
parents:
35548
diff
changeset
|
6460 |
Address::ScaleFactor scale2 = Address::no_scale; |
33628 | 6461 |
|
35113 | 6462 |
if (ae != StrIntrinsicNode::LL) { |
6463 |
stride2x2 = 0x20; |
|
6464 |
} |
|
6465 |
||
33628 | 6466 |
if (ae == StrIntrinsicNode::LU || ae == StrIntrinsicNode::UL) { |
6467 |
shrl(cnt2, 1); |
|
6468 |
} |
|
14626 | 6469 |
// Compute the minimum of the string lengths and the |
6470 |
// difference of the string lengths (stack). |
|
6471 |
// Do the conditional move stuff |
|
6472 |
movl(result, cnt1); |
|
6473 |
subl(cnt1, cnt2); |
|
6474 |
push(cnt1); |
|
35113 | 6475 |
cmov32(Assembler::lessEqual, cnt2, result); // cnt2 = min(cnt1, cnt2) |
14626 | 6476 |
|
6477 |
// Is the minimum length zero? |
|
6478 |
testl(cnt2, cnt2); |
|
6479 |
jcc(Assembler::zero, LENGTH_DIFF_LABEL); |
|
33628 | 6480 |
if (ae == StrIntrinsicNode::LL) { |
6481 |
// Load first bytes |
|
35113 | 6482 |
load_unsigned_byte(result, Address(str1, 0)); // result = str1[0] |
6483 |
load_unsigned_byte(cnt1, Address(str2, 0)); // cnt1 = str2[0] |
|
33628 | 6484 |
} else if (ae == StrIntrinsicNode::UU) { |
6485 |
// Load first characters |
|
6486 |
load_unsigned_short(result, Address(str1, 0)); |
|
6487 |
load_unsigned_short(cnt1, Address(str2, 0)); |
|
6488 |
} else { |
|
6489 |
load_unsigned_byte(result, Address(str1, 0)); |
|
6490 |
load_unsigned_short(cnt1, Address(str2, 0)); |
|
6491 |
} |
|
14626 | 6492 |
subl(result, cnt1); |
6493 |
jcc(Assembler::notZero, POP_LABEL); |
|
33628 | 6494 |
|
6495 |
if (ae == StrIntrinsicNode::UU) { |
|
6496 |
// Divide length by 2 to get number of chars |
|
6497 |
shrl(cnt2, 1); |
|
6498 |
} |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6499 |
cmpl(cnt2, 1); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6500 |
jcc(Assembler::equal, LENGTH_DIFF_LABEL); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6501 |
|
33628 | 6502 |
// Check if the strings start at the same location and setup scale and stride |
6503 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
|
6504 |
cmpptr(str1, str2); |
|
6505 |
jcc(Assembler::equal, LENGTH_DIFF_LABEL); |
|
6506 |
if (ae == StrIntrinsicNode::LL) { |
|
6507 |
scale = Address::times_1; |
|
6508 |
stride = 16; |
|
6509 |
} else { |
|
6510 |
scale = Address::times_2; |
|
6511 |
stride = 8; |
|
6512 |
} |
|
6513 |
} else { |
|
6514 |
scale1 = Address::times_1; |
|
6515 |
scale2 = Address::times_2; |
|
36061
baa556050d22
8145700: Uninitialised variable in macroAssembler_x86.cpp:7038
thartmann
parents:
35548
diff
changeset
|
6516 |
// scale not used |
33628 | 6517 |
stride = 8; |
6518 |
} |
|
14626 | 6519 |
|
15612
d4073ad8ce3d
8007708: compiler/6855215 assert(VM_Version::supports_sse4_2())
kvn
parents:
15483
diff
changeset
|
6520 |
if (UseAVX >= 2 && UseSSE42Intrinsics) { |
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6521 |
Label COMPARE_WIDE_VECTORS, VECTOR_NOT_EQUAL, COMPARE_WIDE_TAIL, COMPARE_SMALL_STR; |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6522 |
Label COMPARE_WIDE_VECTORS_LOOP, COMPARE_16_CHARS, COMPARE_INDEX_CHAR; |
35113 | 6523 |
Label COMPARE_WIDE_VECTORS_LOOP_AVX2; |
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6524 |
Label COMPARE_TAIL_LONG; |
35113 | 6525 |
Label COMPARE_WIDE_VECTORS_LOOP_AVX3; // used only _LP64 && AVX3 |
6526 |
||
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6527 |
int pcmpmask = 0x19; |
33628 | 6528 |
if (ae == StrIntrinsicNode::LL) { |
6529 |
pcmpmask &= ~0x01; |
|
6530 |
} |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6531 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6532 |
// Setup to compare 16-chars (32-bytes) vectors, |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6533 |
// start from first character again because it has aligned address. |
33628 | 6534 |
if (ae == StrIntrinsicNode::LL) { |
6535 |
stride2 = 32; |
|
6536 |
} else { |
|
6537 |
stride2 = 16; |
|
6538 |
} |
|
6539 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
|
6540 |
adr_stride = stride << scale; |
|
6541 |
} else { |
|
6542 |
adr_stride1 = 8; //stride << scale1; |
|
6543 |
adr_stride2 = 16; //stride << scale2; |
|
6544 |
} |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6545 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6546 |
assert(result == rax && cnt2 == rdx && cnt1 == rcx, "pcmpestri"); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6547 |
// rax and rdx are used by pcmpestri as elements counters |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6548 |
movl(result, cnt2); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6549 |
andl(cnt2, ~(stride2-1)); // cnt2 holds the vector count |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6550 |
jcc(Assembler::zero, COMPARE_TAIL_LONG); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6551 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6552 |
// fast path : compare first 2 8-char vectors. |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6553 |
bind(COMPARE_16_CHARS); |
33628 | 6554 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
6555 |
movdqu(vec1, Address(str1, 0)); |
|
6556 |
} else { |
|
6557 |
pmovzxbw(vec1, Address(str1, 0)); |
|
6558 |
} |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6559 |
pcmpestri(vec1, Address(str2, 0), pcmpmask); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6560 |
jccb(Assembler::below, COMPARE_INDEX_CHAR); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6561 |
|
33628 | 6562 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
6563 |
movdqu(vec1, Address(str1, adr_stride)); |
|
6564 |
pcmpestri(vec1, Address(str2, adr_stride), pcmpmask); |
|
6565 |
} else { |
|
6566 |
pmovzxbw(vec1, Address(str1, adr_stride1)); |
|
6567 |
pcmpestri(vec1, Address(str2, adr_stride2), pcmpmask); |
|
6568 |
} |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6569 |
jccb(Assembler::aboveEqual, COMPARE_WIDE_VECTORS); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6570 |
addl(cnt1, stride); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6571 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6572 |
// Compare the characters at index in cnt1 |
33628 | 6573 |
bind(COMPARE_INDEX_CHAR); // cnt1 has the offset of the mismatching character |
6574 |
load_next_elements(result, cnt2, str1, str2, scale, scale1, scale2, cnt1, ae); |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6575 |
subl(result, cnt2); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6576 |
jmp(POP_LABEL); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6577 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6578 |
// Setup the registers to start vector comparison loop |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6579 |
bind(COMPARE_WIDE_VECTORS); |
33628 | 6580 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
6581 |
lea(str1, Address(str1, result, scale)); |
|
6582 |
lea(str2, Address(str2, result, scale)); |
|
6583 |
} else { |
|
6584 |
lea(str1, Address(str1, result, scale1)); |
|
6585 |
lea(str2, Address(str2, result, scale2)); |
|
6586 |
} |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6587 |
subl(result, stride2); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6588 |
subl(cnt2, stride2); |
35113 | 6589 |
jcc(Assembler::zero, COMPARE_WIDE_TAIL); |
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6590 |
negptr(result); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6591 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6592 |
// In a loop, compare 16-chars (32-bytes) at once using (vpxor+vptest) |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6593 |
bind(COMPARE_WIDE_VECTORS_LOOP); |
35113 | 6594 |
|
6595 |
#ifdef _LP64 |
|
58462
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
6596 |
if ((AVX3Threshold == 0) && VM_Version::supports_avx512vlbw()) { // trying 64 bytes fast loop |
35113 | 6597 |
cmpl(cnt2, stride2x2); |
6598 |
jccb(Assembler::below, COMPARE_WIDE_VECTORS_LOOP_AVX2); |
|
6599 |
testl(cnt2, stride2x2-1); // cnt2 holds the vector count |
|
6600 |
jccb(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP_AVX2); // means we cannot subtract by 0x40 |
|
6601 |
||
6602 |
bind(COMPARE_WIDE_VECTORS_LOOP_AVX3); // the hottest loop |
|
6603 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
|
6604 |
evmovdquq(vec1, Address(str1, result, scale), Assembler::AVX_512bit); |
|
6605 |
evpcmpeqb(k7, vec1, Address(str2, result, scale), Assembler::AVX_512bit); // k7 == 11..11, if operands equal, otherwise k7 has some 0 |
|
6606 |
} else { |
|
6607 |
vpmovzxbw(vec1, Address(str1, result, scale1), Assembler::AVX_512bit); |
|
6608 |
evpcmpeqb(k7, vec1, Address(str2, result, scale2), Assembler::AVX_512bit); // k7 == 11..11, if operands equal, otherwise k7 has some 0 |
|
6609 |
} |
|
6610 |
kortestql(k7, k7); |
|
6611 |
jcc(Assembler::aboveEqual, COMPARE_WIDE_VECTORS_LOOP_FAILED); // miscompare |
|
6612 |
addptr(result, stride2x2); // update since we already compared at this addr |
|
6613 |
subl(cnt2, stride2x2); // and sub the size too |
|
6614 |
jccb(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP_AVX3); |
|
6615 |
||
6616 |
vpxor(vec1, vec1); |
|
6617 |
jmpb(COMPARE_WIDE_TAIL); |
|
6618 |
}//if (VM_Version::supports_avx512vlbw()) |
|
6619 |
#endif // _LP64 |
|
6620 |
||
6621 |
||
6622 |
bind(COMPARE_WIDE_VECTORS_LOOP_AVX2); |
|
33628 | 6623 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
6624 |
vmovdqu(vec1, Address(str1, result, scale)); |
|
6625 |
vpxor(vec1, Address(str2, result, scale)); |
|
6626 |
} else { |
|
34203 | 6627 |
vpmovzxbw(vec1, Address(str1, result, scale1), Assembler::AVX_256bit); |
33628 | 6628 |
vpxor(vec1, Address(str2, result, scale2)); |
6629 |
} |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6630 |
vptest(vec1, vec1); |
35113 | 6631 |
jcc(Assembler::notZero, VECTOR_NOT_EQUAL); |
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6632 |
addptr(result, stride2); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6633 |
subl(cnt2, stride2); |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
6634 |
jcc(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP); |
16624
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
6635 |
// clean upper bits of YMM registers |
30299 | 6636 |
vpxor(vec1, vec1); |
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6637 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6638 |
// compare wide vectors tail |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6639 |
bind(COMPARE_WIDE_TAIL); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6640 |
testptr(result, result); |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
6641 |
jcc(Assembler::zero, LENGTH_DIFF_LABEL); |
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6642 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6643 |
movl(result, stride2); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6644 |
movl(cnt2, result); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6645 |
negptr(result); |
35113 | 6646 |
jmp(COMPARE_WIDE_VECTORS_LOOP_AVX2); |
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6647 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6648 |
// Identifies the mismatching (higher or lower)16-bytes in the 32-byte vectors. |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6649 |
bind(VECTOR_NOT_EQUAL); |
16624
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
6650 |
// clean upper bits of YMM registers |
30299 | 6651 |
vpxor(vec1, vec1); |
33628 | 6652 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
6653 |
lea(str1, Address(str1, result, scale)); |
|
6654 |
lea(str2, Address(str2, result, scale)); |
|
6655 |
} else { |
|
6656 |
lea(str1, Address(str1, result, scale1)); |
|
6657 |
lea(str2, Address(str2, result, scale2)); |
|
6658 |
} |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6659 |
jmp(COMPARE_16_CHARS); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6660 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6661 |
// Compare tail chars, length between 1 to 15 chars |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6662 |
bind(COMPARE_TAIL_LONG); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6663 |
movl(cnt2, result); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6664 |
cmpl(cnt2, stride); |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
6665 |
jcc(Assembler::less, COMPARE_SMALL_STR); |
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6666 |
|
33628 | 6667 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
6668 |
movdqu(vec1, Address(str1, 0)); |
|
6669 |
} else { |
|
6670 |
pmovzxbw(vec1, Address(str1, 0)); |
|
6671 |
} |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6672 |
pcmpestri(vec1, Address(str2, 0), pcmpmask); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6673 |
jcc(Assembler::below, COMPARE_INDEX_CHAR); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6674 |
subptr(cnt2, stride); |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
6675 |
jcc(Assembler::zero, LENGTH_DIFF_LABEL); |
33628 | 6676 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
6677 |
lea(str1, Address(str1, result, scale)); |
|
6678 |
lea(str2, Address(str2, result, scale)); |
|
6679 |
} else { |
|
6680 |
lea(str1, Address(str1, result, scale1)); |
|
6681 |
lea(str2, Address(str2, result, scale2)); |
|
6682 |
} |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6683 |
negptr(cnt2); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6684 |
jmpb(WHILE_HEAD_LABEL); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6685 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6686 |
bind(COMPARE_SMALL_STR); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6687 |
} else if (UseSSE42Intrinsics) { |
14626 | 6688 |
Label COMPARE_WIDE_VECTORS, VECTOR_NOT_EQUAL, COMPARE_TAIL; |
6689 |
int pcmpmask = 0x19; |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6690 |
// Setup to compare 8-char (16-byte) vectors, |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6691 |
// start from first character again because it has aligned address. |
14626 | 6692 |
movl(result, cnt2); |
6693 |
andl(cnt2, ~(stride - 1)); // cnt2 holds the vector count |
|
33628 | 6694 |
if (ae == StrIntrinsicNode::LL) { |
6695 |
pcmpmask &= ~0x01; |
|
6696 |
} |
|
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
6697 |
jcc(Assembler::zero, COMPARE_TAIL); |
33628 | 6698 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
6699 |
lea(str1, Address(str1, result, scale)); |
|
6700 |
lea(str2, Address(str2, result, scale)); |
|
6701 |
} else { |
|
6702 |
lea(str1, Address(str1, result, scale1)); |
|
6703 |
lea(str2, Address(str2, result, scale2)); |
|
6704 |
} |
|
14626 | 6705 |
negptr(result); |
6706 |
||
6707 |
// pcmpestri |
|
6708 |
// inputs: |
|
6709 |
// vec1- substring |
|
6710 |
// rax - negative string length (elements count) |
|
28719 | 6711 |
// mem - scanned string |
14626 | 6712 |
// rdx - string length (elements count) |
6713 |
// pcmpmask - cmp mode: 11000 (string compare with negated result) |
|
6714 |
// + 00 (unsigned bytes) or + 01 (unsigned shorts) |
|
6715 |
// outputs: |
|
6716 |
// rcx - first mismatched element index |
|
6717 |
assert(result == rax && cnt2 == rdx && cnt1 == rcx, "pcmpestri"); |
|
6718 |
||
6719 |
bind(COMPARE_WIDE_VECTORS); |
|
33628 | 6720 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
6721 |
movdqu(vec1, Address(str1, result, scale)); |
|
6722 |
pcmpestri(vec1, Address(str2, result, scale), pcmpmask); |
|
6723 |
} else { |
|
6724 |
pmovzxbw(vec1, Address(str1, result, scale1)); |
|
6725 |
pcmpestri(vec1, Address(str2, result, scale2), pcmpmask); |
|
6726 |
} |
|
14626 | 6727 |
// After pcmpestri cnt1(rcx) contains mismatched element index |
6728 |
||
6729 |
jccb(Assembler::below, VECTOR_NOT_EQUAL); // CF==1 |
|
6730 |
addptr(result, stride); |
|
6731 |
subptr(cnt2, stride); |
|
6732 |
jccb(Assembler::notZero, COMPARE_WIDE_VECTORS); |
|
6733 |
||
6734 |
// compare wide vectors tail |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6735 |
testptr(result, result); |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
6736 |
jcc(Assembler::zero, LENGTH_DIFF_LABEL); |
14626 | 6737 |
|
6738 |
movl(cnt2, stride); |
|
6739 |
movl(result, stride); |
|
6740 |
negptr(result); |
|
33628 | 6741 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
6742 |
movdqu(vec1, Address(str1, result, scale)); |
|
6743 |
pcmpestri(vec1, Address(str2, result, scale), pcmpmask); |
|
6744 |
} else { |
|
6745 |
pmovzxbw(vec1, Address(str1, result, scale1)); |
|
6746 |
pcmpestri(vec1, Address(str2, result, scale2), pcmpmask); |
|
6747 |
} |
|
14626 | 6748 |
jccb(Assembler::aboveEqual, LENGTH_DIFF_LABEL); |
6749 |
||
6750 |
// Mismatched characters in the vectors |
|
6751 |
bind(VECTOR_NOT_EQUAL); |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6752 |
addptr(cnt1, result); |
33628 | 6753 |
load_next_elements(result, cnt2, str1, str2, scale, scale1, scale2, cnt1, ae); |
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6754 |
subl(result, cnt2); |
14626 | 6755 |
jmpb(POP_LABEL); |
6756 |
||
6757 |
bind(COMPARE_TAIL); // limit is zero |
|
6758 |
movl(cnt2, result); |
|
6759 |
// Fallthru to tail compare |
|
6760 |
} |
|
6761 |
// Shift str2 and str1 to the end of the arrays, negate min |
|
33628 | 6762 |
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
6763 |
lea(str1, Address(str1, cnt2, scale)); |
|
6764 |
lea(str2, Address(str2, cnt2, scale)); |
|
6765 |
} else { |
|
6766 |
lea(str1, Address(str1, cnt2, scale1)); |
|
6767 |
lea(str2, Address(str2, cnt2, scale2)); |
|
6768 |
} |
|
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
6769 |
decrementl(cnt2); // first character was compared already |
14626 | 6770 |
negptr(cnt2); |
6771 |
||
6772 |
// Compare the rest of the elements |
|
6773 |
bind(WHILE_HEAD_LABEL); |
|
33628 | 6774 |
load_next_elements(result, cnt1, str1, str2, scale, scale1, scale2, cnt2, ae); |
14626 | 6775 |
subl(result, cnt1); |
6776 |
jccb(Assembler::notZero, POP_LABEL); |
|
6777 |
increment(cnt2); |
|
6778 |
jccb(Assembler::notZero, WHILE_HEAD_LABEL); |
|
6779 |
||
6780 |
// Strings are equal up to min length. Return the length difference. |
|
6781 |
bind(LENGTH_DIFF_LABEL); |
|
6782 |
pop(result); |
|
33628 | 6783 |
if (ae == StrIntrinsicNode::UU) { |
6784 |
// Divide diff by 2 to get number of chars |
|
6785 |
sarl(result, 1); |
|
6786 |
} |
|
14626 | 6787 |
jmpb(DONE_LABEL); |
6788 |
||
35113 | 6789 |
#ifdef _LP64 |
6790 |
if (VM_Version::supports_avx512vlbw()) { |
|
6791 |
||
6792 |
bind(COMPARE_WIDE_VECTORS_LOOP_FAILED); |
|
6793 |
||
6794 |
kmovql(cnt1, k7); |
|
6795 |
notq(cnt1); |
|
6796 |
bsfq(cnt2, cnt1); |
|
6797 |
if (ae != StrIntrinsicNode::LL) { |
|
6798 |
// Divide diff by 2 to get number of chars |
|
6799 |
sarl(cnt2, 1); |
|
6800 |
} |
|
6801 |
addq(result, cnt2); |
|
6802 |
if (ae == StrIntrinsicNode::LL) { |
|
6803 |
load_unsigned_byte(cnt1, Address(str2, result)); |
|
6804 |
load_unsigned_byte(result, Address(str1, result)); |
|
6805 |
} else if (ae == StrIntrinsicNode::UU) { |
|
6806 |
load_unsigned_short(cnt1, Address(str2, result, scale)); |
|
6807 |
load_unsigned_short(result, Address(str1, result, scale)); |
|
6808 |
} else { |
|
6809 |
load_unsigned_short(cnt1, Address(str2, result, scale2)); |
|
6810 |
load_unsigned_byte(result, Address(str1, result, scale1)); |
|
6811 |
} |
|
6812 |
subl(result, cnt1); |
|
6813 |
jmpb(POP_LABEL); |
|
6814 |
}//if (VM_Version::supports_avx512vlbw()) |
|
6815 |
#endif // _LP64 |
|
6816 |
||
14626 | 6817 |
// Discard the stored length difference |
6818 |
bind(POP_LABEL); |
|
6819 |
pop(cnt1); |
|
6820 |
||
6821 |
// That's it |
|
6822 |
bind(DONE_LABEL); |
|
33628 | 6823 |
if(ae == StrIntrinsicNode::UL) { |
6824 |
negl(result); |
|
6825 |
} |
|
35113 | 6826 |
|
33628 | 6827 |
} |
6828 |
||
6829 |
// Search for Non-ASCII character (Negative byte value) in a byte array, |
|
6830 |
// return true if it has any and false otherwise. |
|
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6831 |
// ..\jdk\src\java.base\share\classes\java\lang\StringCoding.java |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6832 |
// @HotSpotIntrinsicCandidate |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6833 |
// private static boolean hasNegatives(byte[] ba, int off, int len) { |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6834 |
// for (int i = off; i < off + len; i++) { |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6835 |
// if (ba[i] < 0) { |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6836 |
// return true; |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6837 |
// } |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6838 |
// } |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6839 |
// return false; |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6840 |
// } |
33628 | 6841 |
void MacroAssembler::has_negatives(Register ary1, Register len, |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6842 |
Register result, Register tmp1, |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6843 |
XMMRegister vec1, XMMRegister vec2) { |
33628 | 6844 |
// rsi: byte array |
6845 |
// rcx: len |
|
6846 |
// rax: result |
|
14626 | 6847 |
ShortBranchVerifier sbv(this); |
33628 | 6848 |
assert_different_registers(ary1, len, result, tmp1); |
6849 |
assert_different_registers(vec1, vec2); |
|
6850 |
Label TRUE_LABEL, FALSE_LABEL, DONE, COMPARE_CHAR, COMPARE_VECTORS, COMPARE_BYTE; |
|
6851 |
||
6852 |
// len == 0 |
|
6853 |
testl(len, len); |
|
6854 |
jcc(Assembler::zero, FALSE_LABEL); |
|
6855 |
||
58462
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
6856 |
if ((AVX3Threshold == 0) && (UseAVX > 2) && // AVX512 |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6857 |
VM_Version::supports_avx512vlbw() && |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6858 |
VM_Version::supports_bmi2()) { |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6859 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6860 |
Label test_64_loop, test_tail; |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6861 |
Register tmp3_aliased = len; |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6862 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6863 |
movl(tmp1, len); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6864 |
vpxor(vec2, vec2, vec2, Assembler::AVX_512bit); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6865 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6866 |
andl(tmp1, 64 - 1); // tail count (in chars) 0x3F |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6867 |
andl(len, ~(64 - 1)); // vector count (in chars) |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6868 |
jccb(Assembler::zero, test_tail); |
33628 | 6869 |
|
6870 |
lea(ary1, Address(ary1, len, Address::times_1)); |
|
6871 |
negptr(len); |
|
6872 |
||
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6873 |
bind(test_64_loop); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6874 |
// Check whether our 64 elements of size byte contain negatives |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6875 |
evpcmpgtb(k2, vec2, Address(ary1, len, Address::times_1), Assembler::AVX_512bit); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6876 |
kortestql(k2, k2); |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
6877 |
jcc(Assembler::notZero, TRUE_LABEL); |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6878 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6879 |
addptr(len, 64); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6880 |
jccb(Assembler::notZero, test_64_loop); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6881 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6882 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6883 |
bind(test_tail); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6884 |
// bail out when there is nothing to be done |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6885 |
testl(tmp1, -1); |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
6886 |
jcc(Assembler::zero, FALSE_LABEL); |
33628 | 6887 |
|
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6888 |
// ~(~0 << len) applied up to two times (for 32-bit scenario) |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6889 |
#ifdef _LP64 |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6890 |
mov64(tmp3_aliased, 0xFFFFFFFFFFFFFFFF); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6891 |
shlxq(tmp3_aliased, tmp3_aliased, tmp1); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6892 |
notq(tmp3_aliased); |
51976
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
6893 |
kmovql(k3, tmp3_aliased); |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6894 |
#else |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6895 |
Label k_init; |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6896 |
jmp(k_init); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6897 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6898 |
// We could not read 64-bits from a general purpose register thus we move |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6899 |
// data required to compose 64 1's to the instruction stream |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6900 |
// We emit 64 byte wide series of elements from 0..63 which later on would |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6901 |
// be used as a compare targets with tail count contained in tmp1 register. |
51976
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
6902 |
// Result would be a k register having tmp1 consecutive number or 1 |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6903 |
// counting from least significant bit. |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6904 |
address tmp = pc(); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6905 |
emit_int64(0x0706050403020100); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6906 |
emit_int64(0x0F0E0D0C0B0A0908); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6907 |
emit_int64(0x1716151413121110); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6908 |
emit_int64(0x1F1E1D1C1B1A1918); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6909 |
emit_int64(0x2726252423222120); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6910 |
emit_int64(0x2F2E2D2C2B2A2928); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6911 |
emit_int64(0x3736353433323130); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6912 |
emit_int64(0x3F3E3D3C3B3A3938); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6913 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6914 |
bind(k_init); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6915 |
lea(len, InternalAddress(tmp)); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6916 |
// create mask to test for negative byte inside a vector |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6917 |
evpbroadcastb(vec1, tmp1, Assembler::AVX_512bit); |
51976
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
6918 |
evpcmpgtb(k3, vec1, Address(len, 0), Assembler::AVX_512bit); |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6919 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6920 |
#endif |
51976
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
6921 |
evpcmpgtb(k2, k3, vec2, Address(ary1, 0), Assembler::AVX_512bit); |
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
6922 |
ktestq(k2, k3); |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
6923 |
jcc(Assembler::notZero, TRUE_LABEL); |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6924 |
|
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
6925 |
jmp(FALSE_LABEL); |
41065
d1b98cc38f04
8164989: Inflate and compress intrinsics produce incorrect results with avx512
mcberg
parents:
40644
diff
changeset
|
6926 |
} else { |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6927 |
movl(result, len); // copy |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6928 |
|
58462
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
6929 |
if (UseAVX >= 2 && UseSSE >= 2) { |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6930 |
// With AVX2, use 32-byte vector compare |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6931 |
Label COMPARE_WIDE_VECTORS, COMPARE_TAIL; |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6932 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6933 |
// Compare 32-byte vectors |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6934 |
andl(result, 0x0000001f); // tail count (in bytes) |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6935 |
andl(len, 0xffffffe0); // vector count (in bytes) |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6936 |
jccb(Assembler::zero, COMPARE_TAIL); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6937 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6938 |
lea(ary1, Address(ary1, len, Address::times_1)); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6939 |
negptr(len); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6940 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6941 |
movl(tmp1, 0x80808080); // create mask to test for Unicode chars in vector |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6942 |
movdl(vec2, tmp1); |
51857 | 6943 |
vpbroadcastd(vec2, vec2, Assembler::AVX_256bit); |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6944 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6945 |
bind(COMPARE_WIDE_VECTORS); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6946 |
vmovdqu(vec1, Address(ary1, len, Address::times_1)); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6947 |
vptest(vec1, vec2); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6948 |
jccb(Assembler::notZero, TRUE_LABEL); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6949 |
addptr(len, 32); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6950 |
jcc(Assembler::notZero, COMPARE_WIDE_VECTORS); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6951 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6952 |
testl(result, result); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6953 |
jccb(Assembler::zero, FALSE_LABEL); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6954 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6955 |
vmovdqu(vec1, Address(ary1, result, Address::times_1, -32)); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6956 |
vptest(vec1, vec2); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6957 |
jccb(Assembler::notZero, TRUE_LABEL); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6958 |
jmpb(FALSE_LABEL); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6959 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6960 |
bind(COMPARE_TAIL); // len is zero |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6961 |
movl(len, result); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6962 |
// Fallthru to tail compare |
41065
d1b98cc38f04
8164989: Inflate and compress intrinsics produce incorrect results with avx512
mcberg
parents:
40644
diff
changeset
|
6963 |
} else if (UseSSE42Intrinsics) { |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6964 |
// With SSE4.2, use double quad vector compare |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6965 |
Label COMPARE_WIDE_VECTORS, COMPARE_TAIL; |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6966 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6967 |
// Compare 16-byte vectors |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6968 |
andl(result, 0x0000000f); // tail count (in bytes) |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6969 |
andl(len, 0xfffffff0); // vector count (in bytes) |
51633
21154cb84d2a
8209594: guarantee(this->is8bit(imm8)) failed: Short forward jump exceeds 8-bit offset
kvn
parents:
51464
diff
changeset
|
6970 |
jcc(Assembler::zero, COMPARE_TAIL); |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6971 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6972 |
lea(ary1, Address(ary1, len, Address::times_1)); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6973 |
negptr(len); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6974 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6975 |
movl(tmp1, 0x80808080); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6976 |
movdl(vec2, tmp1); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6977 |
pshufd(vec2, vec2, 0); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6978 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6979 |
bind(COMPARE_WIDE_VECTORS); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6980 |
movdqu(vec1, Address(ary1, len, Address::times_1)); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6981 |
ptest(vec1, vec2); |
51633
21154cb84d2a
8209594: guarantee(this->is8bit(imm8)) failed: Short forward jump exceeds 8-bit offset
kvn
parents:
51464
diff
changeset
|
6982 |
jcc(Assembler::notZero, TRUE_LABEL); |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6983 |
addptr(len, 16); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6984 |
jcc(Assembler::notZero, COMPARE_WIDE_VECTORS); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6985 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6986 |
testl(result, result); |
51633
21154cb84d2a
8209594: guarantee(this->is8bit(imm8)) failed: Short forward jump exceeds 8-bit offset
kvn
parents:
51464
diff
changeset
|
6987 |
jcc(Assembler::zero, FALSE_LABEL); |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6988 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6989 |
movdqu(vec1, Address(ary1, result, Address::times_1, -16)); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6990 |
ptest(vec1, vec2); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6991 |
jccb(Assembler::notZero, TRUE_LABEL); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6992 |
jmpb(FALSE_LABEL); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6993 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6994 |
bind(COMPARE_TAIL); // len is zero |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6995 |
movl(len, result); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6996 |
// Fallthru to tail compare |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6997 |
} |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
6998 |
} |
33628 | 6999 |
// Compare 4-byte vectors |
7000 |
andl(len, 0xfffffffc); // vector count (in bytes) |
|
7001 |
jccb(Assembler::zero, COMPARE_CHAR); |
|
7002 |
||
7003 |
lea(ary1, Address(ary1, len, Address::times_1)); |
|
7004 |
negptr(len); |
|
7005 |
||
7006 |
bind(COMPARE_VECTORS); |
|
7007 |
movl(tmp1, Address(ary1, len, Address::times_1)); |
|
7008 |
andl(tmp1, 0x80808080); |
|
7009 |
jccb(Assembler::notZero, TRUE_LABEL); |
|
7010 |
addptr(len, 4); |
|
7011 |
jcc(Assembler::notZero, COMPARE_VECTORS); |
|
7012 |
||
7013 |
// Compare trailing char (final 2 bytes), if any |
|
7014 |
bind(COMPARE_CHAR); |
|
7015 |
testl(result, 0x2); // tail char |
|
7016 |
jccb(Assembler::zero, COMPARE_BYTE); |
|
7017 |
load_unsigned_short(tmp1, Address(ary1, 0)); |
|
7018 |
andl(tmp1, 0x00008080); |
|
7019 |
jccb(Assembler::notZero, TRUE_LABEL); |
|
7020 |
subptr(result, 2); |
|
7021 |
lea(ary1, Address(ary1, 2)); |
|
7022 |
||
7023 |
bind(COMPARE_BYTE); |
|
7024 |
testl(result, 0x1); // tail byte |
|
7025 |
jccb(Assembler::zero, FALSE_LABEL); |
|
7026 |
load_unsigned_byte(tmp1, Address(ary1, 0)); |
|
7027 |
andl(tmp1, 0x00000080); |
|
7028 |
jccb(Assembler::notEqual, TRUE_LABEL); |
|
7029 |
jmpb(FALSE_LABEL); |
|
7030 |
||
7031 |
bind(TRUE_LABEL); |
|
7032 |
movl(result, 1); // return true |
|
7033 |
jmpb(DONE); |
|
7034 |
||
7035 |
bind(FALSE_LABEL); |
|
7036 |
xorl(result, result); // return false |
|
7037 |
||
7038 |
// That's it |
|
7039 |
bind(DONE); |
|
34207
a5f1c458b56e
8143208: compiler/c1/6855215/Test6855215.java supports_sse2() failed
zmajo
parents:
34203
diff
changeset
|
7040 |
if (UseAVX >= 2 && UseSSE >= 2) { |
33628 | 7041 |
// clean upper bits of YMM registers |
7042 |
vpxor(vec1, vec1); |
|
7043 |
vpxor(vec2, vec2); |
|
7044 |
} |
|
7045 |
} |
|
7046 |
// Compare char[] or byte[] arrays aligned to 4 bytes or substrings. |
|
7047 |
void MacroAssembler::arrays_equals(bool is_array_equ, Register ary1, Register ary2, |
|
7048 |
Register limit, Register result, Register chr, |
|
7049 |
XMMRegister vec1, XMMRegister vec2, bool is_char) { |
|
7050 |
ShortBranchVerifier sbv(this); |
|
7051 |
Label TRUE_LABEL, FALSE_LABEL, DONE, COMPARE_VECTORS, COMPARE_CHAR, COMPARE_BYTE; |
|
14626 | 7052 |
|
7053 |
int length_offset = arrayOopDesc::length_offset_in_bytes(); |
|
33628 | 7054 |
int base_offset = arrayOopDesc::base_offset_in_bytes(is_char ? T_CHAR : T_BYTE); |
14626 | 7055 |
|
7056 |
if (is_array_equ) { |
|
33628 | 7057 |
// Check the input args |
47683
f433d49aceb4
8184914: Use MacroAssembler::cmpoop() consistently when comparing heap objects
rkennke
parents:
47580
diff
changeset
|
7058 |
cmpoop(ary1, ary2); |
33628 | 7059 |
jcc(Assembler::equal, TRUE_LABEL); |
7060 |
||
14626 | 7061 |
// Need additional checks for arrays_equals. |
7062 |
testptr(ary1, ary1); |
|
7063 |
jcc(Assembler::zero, FALSE_LABEL); |
|
7064 |
testptr(ary2, ary2); |
|
7065 |
jcc(Assembler::zero, FALSE_LABEL); |
|
7066 |
||
7067 |
// Check the lengths |
|
7068 |
movl(limit, Address(ary1, length_offset)); |
|
7069 |
cmpl(limit, Address(ary2, length_offset)); |
|
7070 |
jcc(Assembler::notEqual, FALSE_LABEL); |
|
7071 |
} |
|
7072 |
||
7073 |
// count == 0 |
|
7074 |
testl(limit, limit); |
|
7075 |
jcc(Assembler::zero, TRUE_LABEL); |
|
7076 |
||
7077 |
if (is_array_equ) { |
|
7078 |
// Load array address |
|
7079 |
lea(ary1, Address(ary1, base_offset)); |
|
7080 |
lea(ary2, Address(ary2, base_offset)); |
|
7081 |
} |
|
7082 |
||
33628 | 7083 |
if (is_array_equ && is_char) { |
7084 |
// arrays_equals when used for char[]. |
|
7085 |
shll(limit, 1); // byte count != 0 |
|
7086 |
} |
|
14626 | 7087 |
movl(result, limit); // copy |
7088 |
||
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7089 |
if (UseAVX >= 2) { |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7090 |
// With AVX2, use 32-byte vector compare |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7091 |
Label COMPARE_WIDE_VECTORS, COMPARE_TAIL; |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7092 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7093 |
// Compare 32-byte vectors |
33628 | 7094 |
andl(result, 0x0000001f); // tail count (in bytes) |
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7095 |
andl(limit, 0xffffffe0); // vector count (in bytes) |
35136
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7096 |
jcc(Assembler::zero, COMPARE_TAIL); |
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7097 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7098 |
lea(ary1, Address(ary1, limit, Address::times_1)); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7099 |
lea(ary2, Address(ary2, limit, Address::times_1)); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7100 |
negptr(limit); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7101 |
|
35136
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7102 |
#ifdef _LP64 |
58462
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7103 |
if ((AVX3Threshold == 0) && VM_Version::supports_avx512vlbw()) { // trying 64 bytes fast loop |
35136
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7104 |
Label COMPARE_WIDE_VECTORS_LOOP_AVX2, COMPARE_WIDE_VECTORS_LOOP_AVX3; |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7105 |
|
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7106 |
cmpl(limit, -64); |
58462
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7107 |
jcc(Assembler::greater, COMPARE_WIDE_VECTORS_LOOP_AVX2); |
35136
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7108 |
|
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7109 |
bind(COMPARE_WIDE_VECTORS_LOOP_AVX3); // the hottest loop |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7110 |
|
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7111 |
evmovdquq(vec1, Address(ary1, limit, Address::times_1), Assembler::AVX_512bit); |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7112 |
evpcmpeqb(k7, vec1, Address(ary2, limit, Address::times_1), Assembler::AVX_512bit); |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7113 |
kortestql(k7, k7); |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7114 |
jcc(Assembler::aboveEqual, FALSE_LABEL); // miscompare |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7115 |
addptr(limit, 64); // update since we already compared at this addr |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7116 |
cmpl(limit, -64); |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7117 |
jccb(Assembler::lessEqual, COMPARE_WIDE_VECTORS_LOOP_AVX3); |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7118 |
|
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7119 |
// At this point we may still need to compare -limit+result bytes. |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7120 |
// We could execute the next two instruction and just continue via non-wide path: |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7121 |
// cmpl(limit, 0); |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7122 |
// jcc(Assembler::equal, COMPARE_TAIL); // true |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7123 |
// But since we stopped at the points ary{1,2}+limit which are |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7124 |
// not farther than 64 bytes from the ends of arrays ary{1,2}+result |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7125 |
// (|limit| <= 32 and result < 32), |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7126 |
// we may just compare the last 64 bytes. |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7127 |
// |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7128 |
addptr(result, -64); // it is safe, bc we just came from this area |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7129 |
evmovdquq(vec1, Address(ary1, result, Address::times_1), Assembler::AVX_512bit); |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7130 |
evpcmpeqb(k7, vec1, Address(ary2, result, Address::times_1), Assembler::AVX_512bit); |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7131 |
kortestql(k7, k7); |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7132 |
jcc(Assembler::aboveEqual, FALSE_LABEL); // miscompare |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7133 |
|
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7134 |
jmp(TRUE_LABEL); |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7135 |
|
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7136 |
bind(COMPARE_WIDE_VECTORS_LOOP_AVX2); |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7137 |
|
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7138 |
}//if (VM_Version::supports_avx512vlbw()) |
f3788128ff3f
8145717: Use AVX3 instructions for Arrays.equals() intrinsic
kvn
parents:
35135
diff
changeset
|
7139 |
#endif //_LP64 |
58462
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7140 |
bind(COMPARE_WIDE_VECTORS); |
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7141 |
vmovdqu(vec1, Address(ary1, limit, Address::times_1)); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7142 |
vmovdqu(vec2, Address(ary2, limit, Address::times_1)); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7143 |
vpxor(vec1, vec2); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7144 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7145 |
vptest(vec1, vec1); |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
7146 |
jcc(Assembler::notZero, FALSE_LABEL); |
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7147 |
addptr(limit, 32); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7148 |
jcc(Assembler::notZero, COMPARE_WIDE_VECTORS); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7149 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7150 |
testl(result, result); |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
7151 |
jcc(Assembler::zero, TRUE_LABEL); |
15117
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7152 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7153 |
vmovdqu(vec1, Address(ary1, result, Address::times_1, -32)); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7154 |
vmovdqu(vec2, Address(ary2, result, Address::times_1, -32)); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7155 |
vpxor(vec1, vec2); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7156 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7157 |
vptest(vec1, vec1); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7158 |
jccb(Assembler::notZero, FALSE_LABEL); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7159 |
jmpb(TRUE_LABEL); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7160 |
|
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7161 |
bind(COMPARE_TAIL); // limit is zero |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7162 |
movl(limit, result); |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7163 |
// Fallthru to tail compare |
625397df6f4f
8005419: Improve intrinsics code performance on x86 by using AVX2
kvn
parents:
15116
diff
changeset
|
7164 |
} else if (UseSSE42Intrinsics) { |
14626 | 7165 |
// With SSE4.2, use double quad vector compare |
7166 |
Label COMPARE_WIDE_VECTORS, COMPARE_TAIL; |
|
7167 |
||
7168 |
// Compare 16-byte vectors |
|
33628 | 7169 |
andl(result, 0x0000000f); // tail count (in bytes) |
14626 | 7170 |
andl(limit, 0xfffffff0); // vector count (in bytes) |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
7171 |
jcc(Assembler::zero, COMPARE_TAIL); |
14626 | 7172 |
|
7173 |
lea(ary1, Address(ary1, limit, Address::times_1)); |
|
7174 |
lea(ary2, Address(ary2, limit, Address::times_1)); |
|
7175 |
negptr(limit); |
|
7176 |
||
7177 |
bind(COMPARE_WIDE_VECTORS); |
|
7178 |
movdqu(vec1, Address(ary1, limit, Address::times_1)); |
|
7179 |
movdqu(vec2, Address(ary2, limit, Address::times_1)); |
|
7180 |
pxor(vec1, vec2); |
|
7181 |
||
7182 |
ptest(vec1, vec1); |
|
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
7183 |
jcc(Assembler::notZero, FALSE_LABEL); |
14626 | 7184 |
addptr(limit, 16); |
7185 |
jcc(Assembler::notZero, COMPARE_WIDE_VECTORS); |
|
7186 |
||
7187 |
testl(result, result); |
|
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
7188 |
jcc(Assembler::zero, TRUE_LABEL); |
14626 | 7189 |
|
7190 |
movdqu(vec1, Address(ary1, result, Address::times_1, -16)); |
|
7191 |
movdqu(vec2, Address(ary2, result, Address::times_1, -16)); |
|
7192 |
pxor(vec1, vec2); |
|
7193 |
||
7194 |
ptest(vec1, vec1); |
|
7195 |
jccb(Assembler::notZero, FALSE_LABEL); |
|
7196 |
jmpb(TRUE_LABEL); |
|
7197 |
||
7198 |
bind(COMPARE_TAIL); // limit is zero |
|
7199 |
movl(limit, result); |
|
7200 |
// Fallthru to tail compare |
|
7201 |
} |
|
7202 |
||
7203 |
// Compare 4-byte vectors |
|
7204 |
andl(limit, 0xfffffffc); // vector count (in bytes) |
|
7205 |
jccb(Assembler::zero, COMPARE_CHAR); |
|
7206 |
||
7207 |
lea(ary1, Address(ary1, limit, Address::times_1)); |
|
7208 |
lea(ary2, Address(ary2, limit, Address::times_1)); |
|
7209 |
negptr(limit); |
|
7210 |
||
7211 |
bind(COMPARE_VECTORS); |
|
7212 |
movl(chr, Address(ary1, limit, Address::times_1)); |
|
7213 |
cmpl(chr, Address(ary2, limit, Address::times_1)); |
|
7214 |
jccb(Assembler::notEqual, FALSE_LABEL); |
|
7215 |
addptr(limit, 4); |
|
7216 |
jcc(Assembler::notZero, COMPARE_VECTORS); |
|
7217 |
||
7218 |
// Compare trailing char (final 2 bytes), if any |
|
7219 |
bind(COMPARE_CHAR); |
|
7220 |
testl(result, 0x2); // tail char |
|
33628 | 7221 |
jccb(Assembler::zero, COMPARE_BYTE); |
14626 | 7222 |
load_unsigned_short(chr, Address(ary1, 0)); |
7223 |
load_unsigned_short(limit, Address(ary2, 0)); |
|
7224 |
cmpl(chr, limit); |
|
7225 |
jccb(Assembler::notEqual, FALSE_LABEL); |
|
7226 |
||
33628 | 7227 |
if (is_array_equ && is_char) { |
7228 |
bind(COMPARE_BYTE); |
|
7229 |
} else { |
|
7230 |
lea(ary1, Address(ary1, 2)); |
|
7231 |
lea(ary2, Address(ary2, 2)); |
|
7232 |
||
7233 |
bind(COMPARE_BYTE); |
|
7234 |
testl(result, 0x1); // tail byte |
|
7235 |
jccb(Assembler::zero, TRUE_LABEL); |
|
7236 |
load_unsigned_byte(chr, Address(ary1, 0)); |
|
7237 |
load_unsigned_byte(limit, Address(ary2, 0)); |
|
7238 |
cmpl(chr, limit); |
|
7239 |
jccb(Assembler::notEqual, FALSE_LABEL); |
|
7240 |
} |
|
14626 | 7241 |
bind(TRUE_LABEL); |
7242 |
movl(result, 1); // return true |
|
7243 |
jmpb(DONE); |
|
7244 |
||
7245 |
bind(FALSE_LABEL); |
|
7246 |
xorl(result, result); // return false |
|
7247 |
||
7248 |
// That's it |
|
7249 |
bind(DONE); |
|
16624
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
7250 |
if (UseAVX >= 2) { |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
7251 |
// clean upper bits of YMM registers |
30299 | 7252 |
vpxor(vec1, vec1); |
7253 |
vpxor(vec2, vec2); |
|
16624
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
7254 |
} |
14626 | 7255 |
} |
7256 |
||
33628 | 7257 |
#endif |
7258 |
||
14626 | 7259 |
void MacroAssembler::generate_fill(BasicType t, bool aligned, |
7260 |
Register to, Register value, Register count, |
|
7261 |
Register rtmp, XMMRegister xtmp) { |
|
7262 |
ShortBranchVerifier sbv(this); |
|
7263 |
assert_different_registers(to, value, count, rtmp); |
|
51756 | 7264 |
Label L_exit; |
14626 | 7265 |
Label L_fill_2_bytes, L_fill_4_bytes; |
7266 |
||
7267 |
int shift = -1; |
|
7268 |
switch (t) { |
|
7269 |
case T_BYTE: |
|
7270 |
shift = 2; |
|
7271 |
break; |
|
7272 |
case T_SHORT: |
|
7273 |
shift = 1; |
|
7274 |
break; |
|
7275 |
case T_INT: |
|
7276 |
shift = 0; |
|
7277 |
break; |
|
7278 |
default: ShouldNotReachHere(); |
|
7279 |
} |
|
7280 |
||
7281 |
if (t == T_BYTE) { |
|
7282 |
andl(value, 0xff); |
|
7283 |
movl(rtmp, value); |
|
7284 |
shll(rtmp, 8); |
|
7285 |
orl(value, rtmp); |
|
7286 |
} |
|
7287 |
if (t == T_SHORT) { |
|
7288 |
andl(value, 0xffff); |
|
7289 |
} |
|
7290 |
if (t == T_BYTE || t == T_SHORT) { |
|
7291 |
movl(rtmp, value); |
|
7292 |
shll(rtmp, 16); |
|
7293 |
orl(value, rtmp); |
|
7294 |
} |
|
7295 |
||
7296 |
cmpl(count, 2<<shift); // Short arrays (< 8 bytes) fill by element |
|
7297 |
jcc(Assembler::below, L_fill_4_bytes); // use unsigned cmp |
|
7298 |
if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) { |
|
51756 | 7299 |
Label L_skip_align2; |
14626 | 7300 |
// align source address at 4 bytes address boundary |
7301 |
if (t == T_BYTE) { |
|
51756 | 7302 |
Label L_skip_align1; |
14626 | 7303 |
// One byte misalignment happens only for byte arrays |
7304 |
testptr(to, 1); |
|
7305 |
jccb(Assembler::zero, L_skip_align1); |
|
7306 |
movb(Address(to, 0), value); |
|
7307 |
increment(to); |
|
7308 |
decrement(count); |
|
7309 |
BIND(L_skip_align1); |
|
7310 |
} |
|
7311 |
// Two bytes misalignment happens only for byte and short (char) arrays |
|
7312 |
testptr(to, 2); |
|
7313 |
jccb(Assembler::zero, L_skip_align2); |
|
7314 |
movw(Address(to, 0), value); |
|
7315 |
addptr(to, 2); |
|
7316 |
subl(count, 1<<(shift-1)); |
|
7317 |
BIND(L_skip_align2); |
|
7318 |
} |
|
7319 |
if (UseSSE < 2) { |
|
7320 |
Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes; |
|
7321 |
// Fill 32-byte chunks |
|
7322 |
subl(count, 8 << shift); |
|
7323 |
jcc(Assembler::less, L_check_fill_8_bytes); |
|
7324 |
align(16); |
|
7325 |
||
7326 |
BIND(L_fill_32_bytes_loop); |
|
7327 |
||
7328 |
for (int i = 0; i < 32; i += 4) { |
|
7329 |
movl(Address(to, i), value); |
|
7330 |
} |
|
7331 |
||
7332 |
addptr(to, 32); |
|
7333 |
subl(count, 8 << shift); |
|
7334 |
jcc(Assembler::greaterEqual, L_fill_32_bytes_loop); |
|
7335 |
BIND(L_check_fill_8_bytes); |
|
7336 |
addl(count, 8 << shift); |
|
7337 |
jccb(Assembler::zero, L_exit); |
|
7338 |
jmpb(L_fill_8_bytes); |
|
7339 |
||
7340 |
// |
|
7341 |
// length is too short, just fill qwords |
|
7342 |
// |
|
7343 |
BIND(L_fill_8_bytes_loop); |
|
7344 |
movl(Address(to, 0), value); |
|
7345 |
movl(Address(to, 4), value); |
|
7346 |
addptr(to, 8); |
|
7347 |
BIND(L_fill_8_bytes); |
|
7348 |
subl(count, 1 << (shift + 1)); |
|
7349 |
jcc(Assembler::greaterEqual, L_fill_8_bytes_loop); |
|
7350 |
// fall through to fill 4 bytes |
|
7351 |
} else { |
|
7352 |
Label L_fill_32_bytes; |
|
7353 |
if (!UseUnalignedLoadStores) { |
|
7354 |
// align to 8 bytes, we know we are 4 byte aligned to start |
|
7355 |
testptr(to, 4); |
|
7356 |
jccb(Assembler::zero, L_fill_32_bytes); |
|
7357 |
movl(Address(to, 0), value); |
|
7358 |
addptr(to, 4); |
|
7359 |
subl(count, 1<<shift); |
|
7360 |
} |
|
7361 |
BIND(L_fill_32_bytes); |
|
7362 |
{ |
|
7363 |
assert( UseSSE >= 2, "supported cpu only" ); |
|
7364 |
Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes; |
|
7365 |
movdl(xtmp, value); |
|
58462
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7366 |
if (UseAVX >= 2 && UseUnalignedLoadStores) { |
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7367 |
Label L_check_fill_32_bytes; |
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7368 |
if (UseAVX > 2) { |
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7369 |
// Fill 64-byte chunks |
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7370 |
Label L_fill_64_bytes_loop_avx3, L_check_fill_64_bytes_avx2; |
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7371 |
|
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7372 |
// If number of bytes to fill < AVX3Threshold, perform fill using AVX2 |
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7373 |
cmpl(count, AVX3Threshold); |
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7374 |
jccb(Assembler::below, L_check_fill_64_bytes_avx2); |
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7375 |
|
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7376 |
vpbroadcastd(xtmp, xtmp, Assembler::AVX_512bit); |
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7377 |
|
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7378 |
subl(count, 16 << shift); |
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7379 |
jccb(Assembler::less, L_check_fill_32_bytes); |
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7380 |
align(16); |
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7381 |
|
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7382 |
BIND(L_fill_64_bytes_loop_avx3); |
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7383 |
evmovdqul(Address(to, 0), xtmp, Assembler::AVX_512bit); |
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7384 |
addptr(to, 64); |
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7385 |
subl(count, 16 << shift); |
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7386 |
jcc(Assembler::greaterEqual, L_fill_64_bytes_loop_avx3); |
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7387 |
jmpb(L_check_fill_32_bytes); |
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7388 |
|
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7389 |
BIND(L_check_fill_64_bytes_avx2); |
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7390 |
} |
30624 | 7391 |
// Fill 64-byte chunks |
58462
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
7392 |
Label L_fill_64_bytes_loop; |
51857 | 7393 |
vpbroadcastd(xtmp, xtmp, Assembler::AVX_256bit); |
15115
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7394 |
|
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7395 |
subl(count, 16 << shift); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7396 |
jcc(Assembler::less, L_check_fill_32_bytes); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7397 |
align(16); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7398 |
|
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7399 |
BIND(L_fill_64_bytes_loop); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7400 |
vmovdqu(Address(to, 0), xtmp); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7401 |
vmovdqu(Address(to, 32), xtmp); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7402 |
addptr(to, 64); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7403 |
subl(count, 16 << shift); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7404 |
jcc(Assembler::greaterEqual, L_fill_64_bytes_loop); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7405 |
|
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7406 |
BIND(L_check_fill_32_bytes); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7407 |
addl(count, 8 << shift); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7408 |
jccb(Assembler::less, L_check_fill_8_bytes); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7409 |
vmovdqu(Address(to, 0), xtmp); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7410 |
addptr(to, 32); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7411 |
subl(count, 8 << shift); |
16624
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
7412 |
|
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
7413 |
BIND(L_check_fill_8_bytes); |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
7414 |
// clean upper bits of YMM registers |
30299 | 7415 |
movdl(xtmp, value); |
7416 |
pshufd(xtmp, xtmp, 0); |
|
14626 | 7417 |
} else { |
15115
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7418 |
// Fill 32-byte chunks |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7419 |
pshufd(xtmp, xtmp, 0); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7420 |
|
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7421 |
subl(count, 8 << shift); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7422 |
jcc(Assembler::less, L_check_fill_8_bytes); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7423 |
align(16); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7424 |
|
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7425 |
BIND(L_fill_32_bytes_loop); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7426 |
|
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7427 |
if (UseUnalignedLoadStores) { |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7428 |
movdqu(Address(to, 0), xtmp); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7429 |
movdqu(Address(to, 16), xtmp); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7430 |
} else { |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7431 |
movq(Address(to, 0), xtmp); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7432 |
movq(Address(to, 8), xtmp); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7433 |
movq(Address(to, 16), xtmp); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7434 |
movq(Address(to, 24), xtmp); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7435 |
} |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7436 |
|
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7437 |
addptr(to, 32); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7438 |
subl(count, 8 << shift); |
f8ef87f6f07f
8005544: Use 256bit YMM registers in arraycopy stubs on x86
kvn
parents:
15114
diff
changeset
|
7439 |
jcc(Assembler::greaterEqual, L_fill_32_bytes_loop); |
16624
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
7440 |
|
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
7441 |
BIND(L_check_fill_8_bytes); |
14626 | 7442 |
} |
7443 |
addl(count, 8 << shift); |
|
7444 |
jccb(Assembler::zero, L_exit); |
|
7445 |
jmpb(L_fill_8_bytes); |
|
7446 |
||
7447 |
// |
|
7448 |
// length is too short, just fill qwords |
|
7449 |
// |
|
7450 |
BIND(L_fill_8_bytes_loop); |
|
7451 |
movq(Address(to, 0), xtmp); |
|
7452 |
addptr(to, 8); |
|
7453 |
BIND(L_fill_8_bytes); |
|
7454 |
subl(count, 1 << (shift + 1)); |
|
7455 |
jcc(Assembler::greaterEqual, L_fill_8_bytes_loop); |
|
7456 |
} |
|
7457 |
} |
|
7458 |
// fill trailing 4 bytes |
|
7459 |
BIND(L_fill_4_bytes); |
|
7460 |
testl(count, 1<<shift); |
|
7461 |
jccb(Assembler::zero, L_fill_2_bytes); |
|
7462 |
movl(Address(to, 0), value); |
|
7463 |
if (t == T_BYTE || t == T_SHORT) { |
|
51756 | 7464 |
Label L_fill_byte; |
14626 | 7465 |
addptr(to, 4); |
7466 |
BIND(L_fill_2_bytes); |
|
7467 |
// fill trailing 2 bytes |
|
7468 |
testl(count, 1<<(shift-1)); |
|
7469 |
jccb(Assembler::zero, L_fill_byte); |
|
7470 |
movw(Address(to, 0), value); |
|
7471 |
if (t == T_BYTE) { |
|
7472 |
addptr(to, 2); |
|
7473 |
BIND(L_fill_byte); |
|
7474 |
// fill trailing byte |
|
7475 |
testl(count, 1); |
|
7476 |
jccb(Assembler::zero, L_exit); |
|
7477 |
movb(Address(to, 0), value); |
|
7478 |
} else { |
|
7479 |
BIND(L_fill_byte); |
|
7480 |
} |
|
7481 |
} else { |
|
7482 |
BIND(L_fill_2_bytes); |
|
7483 |
} |
|
7484 |
BIND(L_exit); |
|
7485 |
} |
|
15242
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7486 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7487 |
// encode char[] to byte[] in ISO_8859_1 |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
7488 |
//@HotSpotIntrinsicCandidate |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
7489 |
//private static int implEncodeISOArray(byte[] sa, int sp, |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
7490 |
//byte[] da, int dp, int len) { |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
7491 |
// int i = 0; |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
7492 |
// for (; i < len; i++) { |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
7493 |
// char c = StringUTF16.getChar(sa, sp++); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
7494 |
// if (c > '\u00FF') |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
7495 |
// break; |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
7496 |
// da[dp++] = (byte)c; |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
7497 |
// } |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
7498 |
// return i; |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
7499 |
//} |
15242
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7500 |
void MacroAssembler::encode_iso_array(Register src, Register dst, Register len, |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
7501 |
XMMRegister tmp1Reg, XMMRegister tmp2Reg, |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
7502 |
XMMRegister tmp3Reg, XMMRegister tmp4Reg, |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
7503 |
Register tmp5, Register result) { |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
7504 |
|
15242
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7505 |
// rsi: src |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7506 |
// rdi: dst |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7507 |
// rdx: len |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7508 |
// rcx: tmp5 |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7509 |
// rax: result |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7510 |
ShortBranchVerifier sbv(this); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7511 |
assert_different_registers(src, dst, len, tmp5, result); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7512 |
Label L_done, L_copy_1_char, L_copy_1_char_exit; |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7513 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7514 |
// set result |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7515 |
xorl(result, result); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7516 |
// check for zero length |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7517 |
testl(len, len); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7518 |
jcc(Assembler::zero, L_done); |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
7519 |
|
15242
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7520 |
movl(result, len); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7521 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7522 |
// Setup pointers |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7523 |
lea(src, Address(src, len, Address::times_2)); // char[] |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7524 |
lea(dst, Address(dst, len, Address::times_1)); // byte[] |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7525 |
negptr(len); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7526 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7527 |
if (UseSSE42Intrinsics || UseAVX >= 2) { |
51756 | 7528 |
Label L_copy_8_chars, L_copy_8_chars_exit; |
15242
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7529 |
Label L_chars_16_check, L_copy_16_chars, L_copy_16_chars_exit; |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7530 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7531 |
if (UseAVX >= 2) { |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7532 |
Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit; |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7533 |
movl(tmp5, 0xff00ff00); // create mask to test for Unicode chars in vector |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7534 |
movdl(tmp1Reg, tmp5); |
51857 | 7535 |
vpbroadcastd(tmp1Reg, tmp1Reg, Assembler::AVX_256bit); |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
7536 |
jmp(L_chars_32_check); |
15242
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7537 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7538 |
bind(L_copy_32_chars); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7539 |
vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64)); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7540 |
vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32)); |
30624 | 7541 |
vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1); |
15242
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7542 |
vptest(tmp2Reg, tmp1Reg); // check for Unicode chars in vector |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7543 |
jccb(Assembler::notZero, L_copy_32_chars_exit); |
30624 | 7544 |
vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1); |
7545 |
vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector_len */ 1); |
|
15242
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7546 |
vmovdqu(Address(dst, len, Address::times_1, -32), tmp4Reg); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7547 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7548 |
bind(L_chars_32_check); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7549 |
addptr(len, 32); |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
7550 |
jcc(Assembler::lessEqual, L_copy_32_chars); |
15242
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7551 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7552 |
bind(L_copy_32_chars_exit); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7553 |
subptr(len, 16); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7554 |
jccb(Assembler::greater, L_copy_16_chars_exit); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7555 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7556 |
} else if (UseSSE42Intrinsics) { |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7557 |
movl(tmp5, 0xff00ff00); // create mask to test for Unicode chars in vector |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7558 |
movdl(tmp1Reg, tmp5); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7559 |
pshufd(tmp1Reg, tmp1Reg, 0); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7560 |
jmpb(L_chars_16_check); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7561 |
} |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7562 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7563 |
bind(L_copy_16_chars); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7564 |
if (UseAVX >= 2) { |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7565 |
vmovdqu(tmp2Reg, Address(src, len, Address::times_2, -32)); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7566 |
vptest(tmp2Reg, tmp1Reg); |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
7567 |
jcc(Assembler::notZero, L_copy_16_chars_exit); |
30624 | 7568 |
vpackuswb(tmp2Reg, tmp2Reg, tmp1Reg, /* vector_len */ 1); |
7569 |
vpermq(tmp3Reg, tmp2Reg, 0xD8, /* vector_len */ 1); |
|
15242
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7570 |
} else { |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7571 |
if (UseAVX > 0) { |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7572 |
movdqu(tmp3Reg, Address(src, len, Address::times_2, -32)); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7573 |
movdqu(tmp4Reg, Address(src, len, Address::times_2, -16)); |
30624 | 7574 |
vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 0); |
15242
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7575 |
} else { |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7576 |
movdqu(tmp3Reg, Address(src, len, Address::times_2, -32)); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7577 |
por(tmp2Reg, tmp3Reg); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7578 |
movdqu(tmp4Reg, Address(src, len, Address::times_2, -16)); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7579 |
por(tmp2Reg, tmp4Reg); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7580 |
} |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7581 |
ptest(tmp2Reg, tmp1Reg); // check for Unicode chars in vector |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7582 |
jccb(Assembler::notZero, L_copy_16_chars_exit); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7583 |
packuswb(tmp3Reg, tmp4Reg); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7584 |
} |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7585 |
movdqu(Address(dst, len, Address::times_1, -16), tmp3Reg); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7586 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7587 |
bind(L_chars_16_check); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7588 |
addptr(len, 16); |
38134
7435f311b441
8154896: xml.transform fails intermittently on SKX
mcberg
parents:
38049
diff
changeset
|
7589 |
jcc(Assembler::lessEqual, L_copy_16_chars); |
15242
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7590 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7591 |
bind(L_copy_16_chars_exit); |
16624
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
7592 |
if (UseAVX >= 2) { |
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
7593 |
// clean upper bits of YMM registers |
30299 | 7594 |
vpxor(tmp2Reg, tmp2Reg); |
7595 |
vpxor(tmp3Reg, tmp3Reg); |
|
7596 |
vpxor(tmp4Reg, tmp4Reg); |
|
7597 |
movdl(tmp1Reg, tmp5); |
|
7598 |
pshufd(tmp1Reg, tmp1Reg, 0); |
|
16624
9dbd4b210bf9
8011102: Clear AVX registers after return from JNI call
kvn
parents:
16368
diff
changeset
|
7599 |
} |
15242
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7600 |
subptr(len, 8); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7601 |
jccb(Assembler::greater, L_copy_8_chars_exit); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7602 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7603 |
bind(L_copy_8_chars); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7604 |
movdqu(tmp3Reg, Address(src, len, Address::times_2, -16)); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7605 |
ptest(tmp3Reg, tmp1Reg); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7606 |
jccb(Assembler::notZero, L_copy_8_chars_exit); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7607 |
packuswb(tmp3Reg, tmp1Reg); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7608 |
movq(Address(dst, len, Address::times_1, -8), tmp3Reg); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7609 |
addptr(len, 8); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7610 |
jccb(Assembler::lessEqual, L_copy_8_chars); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7611 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7612 |
bind(L_copy_8_chars_exit); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7613 |
subptr(len, 8); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7614 |
jccb(Assembler::zero, L_done); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7615 |
} |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7616 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7617 |
bind(L_copy_1_char); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7618 |
load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0)); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7619 |
testl(tmp5, 0xff00); // check if Unicode char |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7620 |
jccb(Assembler::notZero, L_copy_1_char_exit); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7621 |
movb(Address(dst, len, Address::times_1, 0), tmp5); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7622 |
addptr(len, 1); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7623 |
jccb(Assembler::less, L_copy_1_char); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7624 |
|
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7625 |
bind(L_copy_1_char_exit); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7626 |
addptr(result, len); // len is negative count of not processed elements |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
7627 |
|
15242
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7628 |
bind(L_done); |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7629 |
} |
695bb216be99
6896617: Optimize sun.nio.cs.ISO_8859_1$Encode.encodeArrayLoop() on x86
kvn
parents:
15117
diff
changeset
|
7630 |
|
26434
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7631 |
#ifdef _LP64 |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7632 |
/** |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7633 |
* Helper for multiply_to_len(). |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7634 |
*/ |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7635 |
void MacroAssembler::add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2) { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7636 |
addq(dest_lo, src1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7637 |
adcq(dest_hi, 0); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7638 |
addq(dest_lo, src2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7639 |
adcq(dest_hi, 0); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7640 |
} |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7641 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7642 |
/** |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7643 |
* Multiply 64 bit by 64 bit first loop. |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7644 |
*/ |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7645 |
void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart, |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7646 |
Register y, Register y_idx, Register z, |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7647 |
Register carry, Register product, |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7648 |
Register idx, Register kdx) { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7649 |
// |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7650 |
// jlong carry, x[], y[], z[]; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7651 |
// for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7652 |
// huge_128 product = y[idx] * x[xstart] + carry; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7653 |
// z[kdx] = (jlong)product; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7654 |
// carry = (jlong)(product >>> 64); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7655 |
// } |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7656 |
// z[xstart] = carry; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7657 |
// |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7658 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7659 |
Label L_first_loop, L_first_loop_exit; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7660 |
Label L_one_x, L_one_y, L_multiply; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7661 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7662 |
decrementl(xstart); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7663 |
jcc(Assembler::negative, L_one_x); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7664 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7665 |
movq(x_xstart, Address(x, xstart, Address::times_4, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7666 |
rorq(x_xstart, 32); // convert big-endian to little-endian |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7667 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7668 |
bind(L_first_loop); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7669 |
decrementl(idx); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7670 |
jcc(Assembler::negative, L_first_loop_exit); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7671 |
decrementl(idx); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7672 |
jcc(Assembler::negative, L_one_y); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7673 |
movq(y_idx, Address(y, idx, Address::times_4, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7674 |
rorq(y_idx, 32); // convert big-endian to little-endian |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7675 |
bind(L_multiply); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7676 |
movq(product, x_xstart); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7677 |
mulq(y_idx); // product(rax) * y_idx -> rdx:rax |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7678 |
addq(product, carry); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7679 |
adcq(rdx, 0); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7680 |
subl(kdx, 2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7681 |
movl(Address(z, kdx, Address::times_4, 4), product); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7682 |
shrq(product, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7683 |
movl(Address(z, kdx, Address::times_4, 0), product); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7684 |
movq(carry, rdx); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7685 |
jmp(L_first_loop); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7686 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7687 |
bind(L_one_y); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7688 |
movl(y_idx, Address(y, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7689 |
jmp(L_multiply); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7690 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7691 |
bind(L_one_x); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7692 |
movl(x_xstart, Address(x, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7693 |
jmp(L_first_loop); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7694 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7695 |
bind(L_first_loop_exit); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7696 |
} |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7697 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7698 |
/** |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7699 |
* Multiply 64 bit by 64 bit and add 128 bit. |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7700 |
*/ |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7701 |
void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y, Register z, |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7702 |
Register yz_idx, Register idx, |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7703 |
Register carry, Register product, int offset) { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7704 |
// huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7705 |
// z[kdx] = (jlong)product; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7706 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7707 |
movq(yz_idx, Address(y, idx, Address::times_4, offset)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7708 |
rorq(yz_idx, 32); // convert big-endian to little-endian |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7709 |
movq(product, x_xstart); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7710 |
mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax) |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7711 |
movq(yz_idx, Address(z, idx, Address::times_4, offset)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7712 |
rorq(yz_idx, 32); // convert big-endian to little-endian |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7713 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7714 |
add2_with_carry(rdx, product, carry, yz_idx); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7715 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7716 |
movl(Address(z, idx, Address::times_4, offset+4), product); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7717 |
shrq(product, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7718 |
movl(Address(z, idx, Address::times_4, offset), product); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7719 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7720 |
} |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7721 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7722 |
/** |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7723 |
* Multiply 128 bit by 128 bit. Unrolled inner loop. |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7724 |
*/ |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7725 |
void MacroAssembler::multiply_128_x_128_loop(Register x_xstart, Register y, Register z, |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7726 |
Register yz_idx, Register idx, Register jdx, |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7727 |
Register carry, Register product, |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7728 |
Register carry2) { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7729 |
// jlong carry, x[], y[], z[]; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7730 |
// int kdx = ystart+1; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7731 |
// for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7732 |
// huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7733 |
// z[kdx+idx+1] = (jlong)product; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7734 |
// jlong carry2 = (jlong)(product >>> 64); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7735 |
// product = (y[idx] * x_xstart) + z[kdx+idx] + carry2; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7736 |
// z[kdx+idx] = (jlong)product; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7737 |
// carry = (jlong)(product >>> 64); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7738 |
// } |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7739 |
// idx += 2; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7740 |
// if (idx > 0) { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7741 |
// product = (y[idx] * x_xstart) + z[kdx+idx] + carry; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7742 |
// z[kdx+idx] = (jlong)product; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7743 |
// carry = (jlong)(product >>> 64); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7744 |
// } |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7745 |
// |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7746 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7747 |
Label L_third_loop, L_third_loop_exit, L_post_third_loop_done; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7748 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7749 |
movl(jdx, idx); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7750 |
andl(jdx, 0xFFFFFFFC); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7751 |
shrl(jdx, 2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7752 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7753 |
bind(L_third_loop); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7754 |
subl(jdx, 1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7755 |
jcc(Assembler::negative, L_third_loop_exit); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7756 |
subl(idx, 4); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7757 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7758 |
multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 8); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7759 |
movq(carry2, rdx); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7760 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7761 |
multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product, 0); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7762 |
movq(carry, rdx); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7763 |
jmp(L_third_loop); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7764 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7765 |
bind (L_third_loop_exit); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7766 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7767 |
andl (idx, 0x3); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7768 |
jcc(Assembler::zero, L_post_third_loop_done); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7769 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7770 |
Label L_check_1; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7771 |
subl(idx, 2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7772 |
jcc(Assembler::negative, L_check_1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7773 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7774 |
multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 0); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7775 |
movq(carry, rdx); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7776 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7777 |
bind (L_check_1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7778 |
addl (idx, 0x2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7779 |
andl (idx, 0x1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7780 |
subl(idx, 1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7781 |
jcc(Assembler::negative, L_post_third_loop_done); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7782 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7783 |
movl(yz_idx, Address(y, idx, Address::times_4, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7784 |
movq(product, x_xstart); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7785 |
mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax) |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7786 |
movl(yz_idx, Address(z, idx, Address::times_4, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7787 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7788 |
add2_with_carry(rdx, product, yz_idx, carry); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7789 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7790 |
movl(Address(z, idx, Address::times_4, 0), product); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7791 |
shrq(product, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7792 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7793 |
shlq(rdx, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7794 |
orq(product, rdx); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7795 |
movq(carry, product); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7796 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7797 |
bind(L_post_third_loop_done); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7798 |
} |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7799 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7800 |
/** |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7801 |
* Multiply 128 bit by 128 bit using BMI2. Unrolled inner loop. |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7802 |
* |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7803 |
*/ |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7804 |
void MacroAssembler::multiply_128_x_128_bmi2_loop(Register y, Register z, |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7805 |
Register carry, Register carry2, |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7806 |
Register idx, Register jdx, |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7807 |
Register yz_idx1, Register yz_idx2, |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7808 |
Register tmp, Register tmp3, Register tmp4) { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7809 |
assert(UseBMI2Instructions, "should be used only when BMI2 is available"); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7810 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7811 |
// jlong carry, x[], y[], z[]; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7812 |
// int kdx = ystart+1; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7813 |
// for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7814 |
// huge_128 tmp3 = (y[idx+1] * rdx) + z[kdx+idx+1] + carry; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7815 |
// jlong carry2 = (jlong)(tmp3 >>> 64); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7816 |
// huge_128 tmp4 = (y[idx] * rdx) + z[kdx+idx] + carry2; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7817 |
// carry = (jlong)(tmp4 >>> 64); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7818 |
// z[kdx+idx+1] = (jlong)tmp3; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7819 |
// z[kdx+idx] = (jlong)tmp4; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7820 |
// } |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7821 |
// idx += 2; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7822 |
// if (idx > 0) { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7823 |
// yz_idx1 = (y[idx] * rdx) + z[kdx+idx] + carry; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7824 |
// z[kdx+idx] = (jlong)yz_idx1; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7825 |
// carry = (jlong)(yz_idx1 >>> 64); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7826 |
// } |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7827 |
// |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7828 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7829 |
Label L_third_loop, L_third_loop_exit, L_post_third_loop_done; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7830 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7831 |
movl(jdx, idx); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7832 |
andl(jdx, 0xFFFFFFFC); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7833 |
shrl(jdx, 2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7834 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7835 |
bind(L_third_loop); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7836 |
subl(jdx, 1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7837 |
jcc(Assembler::negative, L_third_loop_exit); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7838 |
subl(idx, 4); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7839 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7840 |
movq(yz_idx1, Address(y, idx, Address::times_4, 8)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7841 |
rorxq(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7842 |
movq(yz_idx2, Address(y, idx, Address::times_4, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7843 |
rorxq(yz_idx2, yz_idx2, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7844 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7845 |
mulxq(tmp4, tmp3, yz_idx1); // yz_idx1 * rdx -> tmp4:tmp3 |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7846 |
mulxq(carry2, tmp, yz_idx2); // yz_idx2 * rdx -> carry2:tmp |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7847 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7848 |
movq(yz_idx1, Address(z, idx, Address::times_4, 8)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7849 |
rorxq(yz_idx1, yz_idx1, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7850 |
movq(yz_idx2, Address(z, idx, Address::times_4, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7851 |
rorxq(yz_idx2, yz_idx2, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7852 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7853 |
if (VM_Version::supports_adx()) { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7854 |
adcxq(tmp3, carry); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7855 |
adoxq(tmp3, yz_idx1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7856 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7857 |
adcxq(tmp4, tmp); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7858 |
adoxq(tmp4, yz_idx2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7859 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7860 |
movl(carry, 0); // does not affect flags |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7861 |
adcxq(carry2, carry); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7862 |
adoxq(carry2, carry); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7863 |
} else { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7864 |
add2_with_carry(tmp4, tmp3, carry, yz_idx1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7865 |
add2_with_carry(carry2, tmp4, tmp, yz_idx2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7866 |
} |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7867 |
movq(carry, carry2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7868 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7869 |
movl(Address(z, idx, Address::times_4, 12), tmp3); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7870 |
shrq(tmp3, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7871 |
movl(Address(z, idx, Address::times_4, 8), tmp3); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7872 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7873 |
movl(Address(z, idx, Address::times_4, 4), tmp4); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7874 |
shrq(tmp4, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7875 |
movl(Address(z, idx, Address::times_4, 0), tmp4); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7876 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7877 |
jmp(L_third_loop); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7878 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7879 |
bind (L_third_loop_exit); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7880 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7881 |
andl (idx, 0x3); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7882 |
jcc(Assembler::zero, L_post_third_loop_done); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7883 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7884 |
Label L_check_1; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7885 |
subl(idx, 2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7886 |
jcc(Assembler::negative, L_check_1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7887 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7888 |
movq(yz_idx1, Address(y, idx, Address::times_4, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7889 |
rorxq(yz_idx1, yz_idx1, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7890 |
mulxq(tmp4, tmp3, yz_idx1); // yz_idx1 * rdx -> tmp4:tmp3 |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7891 |
movq(yz_idx2, Address(z, idx, Address::times_4, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7892 |
rorxq(yz_idx2, yz_idx2, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7893 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7894 |
add2_with_carry(tmp4, tmp3, carry, yz_idx2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7895 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7896 |
movl(Address(z, idx, Address::times_4, 4), tmp3); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7897 |
shrq(tmp3, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7898 |
movl(Address(z, idx, Address::times_4, 0), tmp3); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7899 |
movq(carry, tmp4); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7900 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7901 |
bind (L_check_1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7902 |
addl (idx, 0x2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7903 |
andl (idx, 0x1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7904 |
subl(idx, 1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7905 |
jcc(Assembler::negative, L_post_third_loop_done); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7906 |
movl(tmp4, Address(y, idx, Address::times_4, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7907 |
mulxq(carry2, tmp3, tmp4); // tmp4 * rdx -> carry2:tmp3 |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7908 |
movl(tmp4, Address(z, idx, Address::times_4, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7909 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7910 |
add2_with_carry(carry2, tmp3, tmp4, carry); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7911 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7912 |
movl(Address(z, idx, Address::times_4, 0), tmp3); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7913 |
shrq(tmp3, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7914 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7915 |
shlq(carry2, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7916 |
orq(tmp3, carry2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7917 |
movq(carry, tmp3); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7918 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7919 |
bind(L_post_third_loop_done); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7920 |
} |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7921 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7922 |
/** |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7923 |
* Code for BigInteger::multiplyToLen() instrinsic. |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7924 |
* |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7925 |
* rdi: x |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7926 |
* rax: xlen |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7927 |
* rsi: y |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7928 |
* rcx: ylen |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7929 |
* r8: z |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7930 |
* r11: zlen |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7931 |
* r12: tmp1 |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7932 |
* r13: tmp2 |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7933 |
* r14: tmp3 |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7934 |
* r15: tmp4 |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7935 |
* rbx: tmp5 |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7936 |
* |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7937 |
*/ |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7938 |
void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register zlen, |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7939 |
Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7940 |
ShortBranchVerifier sbv(this); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7941 |
assert_different_registers(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, rdx); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7942 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7943 |
push(tmp1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7944 |
push(tmp2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7945 |
push(tmp3); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7946 |
push(tmp4); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7947 |
push(tmp5); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7948 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7949 |
push(xlen); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7950 |
push(zlen); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7951 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7952 |
const Register idx = tmp1; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7953 |
const Register kdx = tmp2; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7954 |
const Register xstart = tmp3; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7955 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7956 |
const Register y_idx = tmp4; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7957 |
const Register carry = tmp5; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7958 |
const Register product = xlen; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7959 |
const Register x_xstart = zlen; // reuse register |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7960 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7961 |
// First Loop. |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7962 |
// |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7963 |
// final static long LONG_MASK = 0xffffffffL; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7964 |
// int xstart = xlen - 1; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7965 |
// int ystart = ylen - 1; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7966 |
// long carry = 0; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7967 |
// for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7968 |
// long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7969 |
// z[kdx] = (int)product; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7970 |
// carry = product >>> 32; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7971 |
// } |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7972 |
// z[xstart] = (int)carry; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7973 |
// |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7974 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7975 |
movl(idx, ylen); // idx = ylen; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7976 |
movl(kdx, zlen); // kdx = xlen+ylen; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7977 |
xorq(carry, carry); // carry = 0; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7978 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7979 |
Label L_done; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7980 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7981 |
movl(xstart, xlen); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7982 |
decrementl(xstart); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7983 |
jcc(Assembler::negative, L_done); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7984 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7985 |
multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7986 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7987 |
Label L_second_loop; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7988 |
testl(kdx, kdx); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7989 |
jcc(Assembler::zero, L_second_loop); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7990 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7991 |
Label L_carry; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7992 |
subl(kdx, 1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7993 |
jcc(Assembler::zero, L_carry); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7994 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7995 |
movl(Address(z, kdx, Address::times_4, 0), carry); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7996 |
shrq(carry, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7997 |
subl(kdx, 1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7998 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
7999 |
bind(L_carry); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8000 |
movl(Address(z, kdx, Address::times_4, 0), carry); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8001 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8002 |
// Second and third (nested) loops. |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8003 |
// |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8004 |
// for (int i = xstart-1; i >= 0; i--) { // Second loop |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8005 |
// carry = 0; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8006 |
// for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8007 |
// long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) + |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8008 |
// (z[k] & LONG_MASK) + carry; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8009 |
// z[k] = (int)product; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8010 |
// carry = product >>> 32; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8011 |
// } |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8012 |
// z[i] = (int)carry; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8013 |
// } |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8014 |
// |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8015 |
// i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8016 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8017 |
const Register jdx = tmp1; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8018 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8019 |
bind(L_second_loop); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8020 |
xorl(carry, carry); // carry = 0; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8021 |
movl(jdx, ylen); // j = ystart+1 |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8022 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8023 |
subl(xstart, 1); // i = xstart-1; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8024 |
jcc(Assembler::negative, L_done); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8025 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8026 |
push (z); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8027 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8028 |
Label L_last_x; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8029 |
lea(z, Address(z, xstart, Address::times_4, 4)); // z = z + k - j |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8030 |
subl(xstart, 1); // i = xstart-1; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8031 |
jcc(Assembler::negative, L_last_x); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8032 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8033 |
if (UseBMI2Instructions) { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8034 |
movq(rdx, Address(x, xstart, Address::times_4, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8035 |
rorxq(rdx, rdx, 32); // convert big-endian to little-endian |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8036 |
} else { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8037 |
movq(x_xstart, Address(x, xstart, Address::times_4, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8038 |
rorq(x_xstart, 32); // convert big-endian to little-endian |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8039 |
} |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8040 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8041 |
Label L_third_loop_prologue; |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8042 |
bind(L_third_loop_prologue); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8043 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8044 |
push (x); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8045 |
push (xstart); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8046 |
push (ylen); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8047 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8048 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8049 |
if (UseBMI2Instructions) { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8050 |
multiply_128_x_128_bmi2_loop(y, z, carry, x, jdx, ylen, product, tmp2, x_xstart, tmp3, tmp4); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8051 |
} else { // !UseBMI2Instructions |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8052 |
multiply_128_x_128_loop(x_xstart, y, z, y_idx, jdx, ylen, carry, product, x); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8053 |
} |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8054 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8055 |
pop(ylen); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8056 |
pop(xlen); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8057 |
pop(x); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8058 |
pop(z); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8059 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8060 |
movl(tmp3, xlen); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8061 |
addl(tmp3, 1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8062 |
movl(Address(z, tmp3, Address::times_4, 0), carry); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8063 |
subl(tmp3, 1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8064 |
jccb(Assembler::negative, L_done); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8065 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8066 |
shrq(carry, 32); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8067 |
movl(Address(z, tmp3, Address::times_4, 0), carry); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8068 |
jmp(L_second_loop); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8069 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8070 |
// Next infrequent code is moved outside loops. |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8071 |
bind(L_last_x); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8072 |
if (UseBMI2Instructions) { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8073 |
movl(rdx, Address(x, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8074 |
} else { |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8075 |
movl(x_xstart, Address(x, 0)); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8076 |
} |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8077 |
jmp(L_third_loop_prologue); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8078 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8079 |
bind(L_done); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8080 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8081 |
pop(zlen); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8082 |
pop(xlen); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8083 |
|
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8084 |
pop(tmp5); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8085 |
pop(tmp4); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8086 |
pop(tmp3); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8087 |
pop(tmp2); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8088 |
pop(tmp1); |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8089 |
} |
31129
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8090 |
|
35110
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8091 |
void MacroAssembler::vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale, |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8092 |
Register result, Register tmp1, Register tmp2, XMMRegister rymm0, XMMRegister rymm1, XMMRegister rymm2){ |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8093 |
assert(UseSSE42Intrinsics, "SSE4.2 must be enabled."); |
51756 | 8094 |
Label VECTOR16_LOOP, VECTOR8_LOOP, VECTOR4_LOOP; |
8095 |
Label VECTOR8_TAIL, VECTOR4_TAIL; |
|
35110
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8096 |
Label VECTOR32_NOT_EQUAL, VECTOR16_NOT_EQUAL, VECTOR8_NOT_EQUAL, VECTOR4_NOT_EQUAL; |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8097 |
Label SAME_TILL_END, DONE; |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8098 |
Label BYTES_LOOP, BYTES_TAIL, BYTES_NOT_EQUAL; |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8099 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8100 |
//scale is in rcx in both Win64 and Unix |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8101 |
ShortBranchVerifier sbv(this); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8102 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8103 |
shlq(length); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8104 |
xorq(result, result); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8105 |
|
58462
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
8106 |
if ((AVX3Threshold == 0) && (UseAVX > 2) && |
38138
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8107 |
VM_Version::supports_avx512vlbw()) { |
51756 | 8108 |
Label VECTOR64_LOOP, VECTOR64_NOT_EQUAL, VECTOR32_TAIL; |
8109 |
||
38138
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8110 |
cmpq(length, 64); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8111 |
jcc(Assembler::less, VECTOR32_TAIL); |
58462
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
8112 |
|
38138
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8113 |
movq(tmp1, length); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8114 |
andq(tmp1, 0x3F); // tail count |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8115 |
andq(length, ~(0x3F)); //vector count |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8116 |
|
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8117 |
bind(VECTOR64_LOOP); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8118 |
// AVX512 code to compare 64 byte vectors. |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8119 |
evmovdqub(rymm0, Address(obja, result), Assembler::AVX_512bit); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8120 |
evpcmpeqb(k7, rymm0, Address(objb, result), Assembler::AVX_512bit); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8121 |
kortestql(k7, k7); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8122 |
jcc(Assembler::aboveEqual, VECTOR64_NOT_EQUAL); // mismatch |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8123 |
addq(result, 64); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8124 |
subq(length, 64); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8125 |
jccb(Assembler::notZero, VECTOR64_LOOP); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8126 |
|
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8127 |
//bind(VECTOR64_TAIL); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8128 |
testq(tmp1, tmp1); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8129 |
jcc(Assembler::zero, SAME_TILL_END); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8130 |
|
51756 | 8131 |
//bind(VECTOR64_TAIL); |
38138
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8132 |
// AVX512 code to compare upto 63 byte vectors. |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8133 |
mov64(tmp2, 0xFFFFFFFFFFFFFFFF); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8134 |
shlxq(tmp2, tmp2, tmp1); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8135 |
notq(tmp2); |
51976
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
8136 |
kmovql(k3, tmp2); |
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
8137 |
|
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
8138 |
evmovdqub(rymm0, k3, Address(obja, result), Assembler::AVX_512bit); |
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
8139 |
evpcmpeqb(k7, k3, rymm0, Address(objb, result), Assembler::AVX_512bit); |
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
8140 |
|
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
8141 |
ktestql(k7, k3); |
38138
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8142 |
jcc(Assembler::below, SAME_TILL_END); // not mismatch |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8143 |
|
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8144 |
bind(VECTOR64_NOT_EQUAL); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8145 |
kmovql(tmp1, k7); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8146 |
notq(tmp1); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8147 |
tzcntq(tmp1, tmp1); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8148 |
addq(result, tmp1); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8149 |
shrq(result); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8150 |
jmp(DONE); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8151 |
bind(VECTOR32_TAIL); |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8152 |
} |
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8153 |
|
35110
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8154 |
cmpq(length, 8); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8155 |
jcc(Assembler::equal, VECTOR8_LOOP); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8156 |
jcc(Assembler::less, VECTOR4_TAIL); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8157 |
|
38138
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8158 |
if (UseAVX >= 2) { |
51756 | 8159 |
Label VECTOR16_TAIL, VECTOR32_LOOP; |
35110
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8160 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8161 |
cmpq(length, 16); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8162 |
jcc(Assembler::equal, VECTOR16_LOOP); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8163 |
jcc(Assembler::less, VECTOR8_LOOP); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8164 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8165 |
cmpq(length, 32); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8166 |
jccb(Assembler::less, VECTOR16_TAIL); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8167 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8168 |
subq(length, 32); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8169 |
bind(VECTOR32_LOOP); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8170 |
vmovdqu(rymm0, Address(obja, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8171 |
vmovdqu(rymm1, Address(objb, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8172 |
vpxor(rymm2, rymm0, rymm1, Assembler::AVX_256bit); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8173 |
vptest(rymm2, rymm2); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8174 |
jcc(Assembler::notZero, VECTOR32_NOT_EQUAL);//mismatch found |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8175 |
addq(result, 32); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8176 |
subq(length, 32); |
51633
21154cb84d2a
8209594: guarantee(this->is8bit(imm8)) failed: Short forward jump exceeds 8-bit offset
kvn
parents:
51464
diff
changeset
|
8177 |
jcc(Assembler::greaterEqual, VECTOR32_LOOP); |
35110
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8178 |
addq(length, 32); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8179 |
jcc(Assembler::equal, SAME_TILL_END); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8180 |
//falling through if less than 32 bytes left //close the branch here. |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8181 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8182 |
bind(VECTOR16_TAIL); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8183 |
cmpq(length, 16); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8184 |
jccb(Assembler::less, VECTOR8_TAIL); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8185 |
bind(VECTOR16_LOOP); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8186 |
movdqu(rymm0, Address(obja, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8187 |
movdqu(rymm1, Address(objb, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8188 |
vpxor(rymm2, rymm0, rymm1, Assembler::AVX_128bit); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8189 |
ptest(rymm2, rymm2); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8190 |
jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8191 |
addq(result, 16); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8192 |
subq(length, 16); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8193 |
jcc(Assembler::equal, SAME_TILL_END); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8194 |
//falling through if less than 16 bytes left |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8195 |
} else {//regular intrinsics |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8196 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8197 |
cmpq(length, 16); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8198 |
jccb(Assembler::less, VECTOR8_TAIL); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8199 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8200 |
subq(length, 16); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8201 |
bind(VECTOR16_LOOP); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8202 |
movdqu(rymm0, Address(obja, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8203 |
movdqu(rymm1, Address(objb, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8204 |
pxor(rymm0, rymm1); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8205 |
ptest(rymm0, rymm0); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8206 |
jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8207 |
addq(result, 16); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8208 |
subq(length, 16); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8209 |
jccb(Assembler::greaterEqual, VECTOR16_LOOP); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8210 |
addq(length, 16); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8211 |
jcc(Assembler::equal, SAME_TILL_END); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8212 |
//falling through if less than 16 bytes left |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8213 |
} |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8214 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8215 |
bind(VECTOR8_TAIL); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8216 |
cmpq(length, 8); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8217 |
jccb(Assembler::less, VECTOR4_TAIL); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8218 |
bind(VECTOR8_LOOP); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8219 |
movq(tmp1, Address(obja, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8220 |
movq(tmp2, Address(objb, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8221 |
xorq(tmp1, tmp2); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8222 |
testq(tmp1, tmp1); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8223 |
jcc(Assembler::notZero, VECTOR8_NOT_EQUAL);//mismatch found |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8224 |
addq(result, 8); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8225 |
subq(length, 8); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8226 |
jcc(Assembler::equal, SAME_TILL_END); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8227 |
//falling through if less than 8 bytes left |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8228 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8229 |
bind(VECTOR4_TAIL); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8230 |
cmpq(length, 4); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8231 |
jccb(Assembler::less, BYTES_TAIL); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8232 |
bind(VECTOR4_LOOP); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8233 |
movl(tmp1, Address(obja, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8234 |
xorl(tmp1, Address(objb, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8235 |
testl(tmp1, tmp1); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8236 |
jcc(Assembler::notZero, VECTOR4_NOT_EQUAL);//mismatch found |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8237 |
addq(result, 4); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8238 |
subq(length, 4); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8239 |
jcc(Assembler::equal, SAME_TILL_END); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8240 |
//falling through if less than 4 bytes left |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8241 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8242 |
bind(BYTES_TAIL); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8243 |
bind(BYTES_LOOP); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8244 |
load_unsigned_byte(tmp1, Address(obja, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8245 |
load_unsigned_byte(tmp2, Address(objb, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8246 |
xorl(tmp1, tmp2); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8247 |
testl(tmp1, tmp1); |
51633
21154cb84d2a
8209594: guarantee(this->is8bit(imm8)) failed: Short forward jump exceeds 8-bit offset
kvn
parents:
51464
diff
changeset
|
8248 |
jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found |
35110
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8249 |
decq(length); |
51633
21154cb84d2a
8209594: guarantee(this->is8bit(imm8)) failed: Short forward jump exceeds 8-bit offset
kvn
parents:
51464
diff
changeset
|
8250 |
jcc(Assembler::zero, SAME_TILL_END); |
35110
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8251 |
incq(result); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8252 |
load_unsigned_byte(tmp1, Address(obja, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8253 |
load_unsigned_byte(tmp2, Address(objb, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8254 |
xorl(tmp1, tmp2); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8255 |
testl(tmp1, tmp1); |
51633
21154cb84d2a
8209594: guarantee(this->is8bit(imm8)) failed: Short forward jump exceeds 8-bit offset
kvn
parents:
51464
diff
changeset
|
8256 |
jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found |
35110
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8257 |
decq(length); |
51633
21154cb84d2a
8209594: guarantee(this->is8bit(imm8)) failed: Short forward jump exceeds 8-bit offset
kvn
parents:
51464
diff
changeset
|
8258 |
jcc(Assembler::zero, SAME_TILL_END); |
35110
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8259 |
incq(result); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8260 |
load_unsigned_byte(tmp1, Address(obja, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8261 |
load_unsigned_byte(tmp2, Address(objb, result)); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8262 |
xorl(tmp1, tmp2); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8263 |
testl(tmp1, tmp1); |
51633
21154cb84d2a
8209594: guarantee(this->is8bit(imm8)) failed: Short forward jump exceeds 8-bit offset
kvn
parents:
51464
diff
changeset
|
8264 |
jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found |
21154cb84d2a
8209594: guarantee(this->is8bit(imm8)) failed: Short forward jump exceeds 8-bit offset
kvn
parents:
51464
diff
changeset
|
8265 |
jmp(SAME_TILL_END); |
35110
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8266 |
|
38138
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8267 |
if (UseAVX >= 2) { |
35110
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8268 |
bind(VECTOR32_NOT_EQUAL); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8269 |
vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_256bit); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8270 |
vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_256bit); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8271 |
vpxor(rymm0, rymm0, rymm2, Assembler::AVX_256bit); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8272 |
vpmovmskb(tmp1, rymm0); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8273 |
bsfq(tmp1, tmp1); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8274 |
addq(result, tmp1); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8275 |
shrq(result); |
51633
21154cb84d2a
8209594: guarantee(this->is8bit(imm8)) failed: Short forward jump exceeds 8-bit offset
kvn
parents:
51464
diff
changeset
|
8276 |
jmp(DONE); |
35110
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8277 |
} |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8278 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8279 |
bind(VECTOR16_NOT_EQUAL); |
38138
8514e24123c8
8154975: Update for vectorizedMismatch with AVX512
vdeshpande
parents:
38134
diff
changeset
|
8280 |
if (UseAVX >= 2) { |
35110
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8281 |
vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_128bit); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8282 |
vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_128bit); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8283 |
pxor(rymm0, rymm2); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8284 |
} else { |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8285 |
pcmpeqb(rymm2, rymm2); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8286 |
pxor(rymm0, rymm1); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8287 |
pcmpeqb(rymm0, rymm1); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8288 |
pxor(rymm0, rymm2); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8289 |
} |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8290 |
pmovmskb(tmp1, rymm0); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8291 |
bsfq(tmp1, tmp1); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8292 |
addq(result, tmp1); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8293 |
shrq(result); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8294 |
jmpb(DONE); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8295 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8296 |
bind(VECTOR8_NOT_EQUAL); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8297 |
bind(VECTOR4_NOT_EQUAL); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8298 |
bsfq(tmp1, tmp1); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8299 |
shrq(tmp1, 3); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8300 |
addq(result, tmp1); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8301 |
bind(BYTES_NOT_EQUAL); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8302 |
shrq(result); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8303 |
jmpb(DONE); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8304 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8305 |
bind(SAME_TILL_END); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8306 |
mov64(result, -1); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8307 |
|
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8308 |
bind(DONE); |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8309 |
} |
f19bcdf40799
8143355: Update for addition of vectorizedMismatch intrinsic for x86
kvn
parents:
35086
diff
changeset
|
8310 |
|
31129
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8311 |
//Helper functions for square_to_len() |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8312 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8313 |
/** |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8314 |
* Store the squares of x[], right shifted one bit (divided by 2) into z[] |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8315 |
* Preserves x and z and modifies rest of the registers. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8316 |
*/ |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8317 |
void MacroAssembler::square_rshift(Register x, Register xlen, Register z, Register tmp1, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8318 |
// Perform square and right shift by 1 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8319 |
// Handle odd xlen case first, then for even xlen do the following |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8320 |
// jlong carry = 0; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8321 |
// for (int j=0, i=0; j < xlen; j+=2, i+=4) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8322 |
// huge_128 product = x[j:j+1] * x[j:j+1]; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8323 |
// z[i:i+1] = (carry << 63) | (jlong)(product >>> 65); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8324 |
// z[i+2:i+3] = (jlong)(product >>> 1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8325 |
// carry = (jlong)product; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8326 |
// } |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8327 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8328 |
xorq(tmp5, tmp5); // carry |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8329 |
xorq(rdxReg, rdxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8330 |
xorl(tmp1, tmp1); // index for x |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8331 |
xorl(tmp4, tmp4); // index for z |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8332 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8333 |
Label L_first_loop, L_first_loop_exit; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8334 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8335 |
testl(xlen, 1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8336 |
jccb(Assembler::zero, L_first_loop); //jump if xlen is even |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8337 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8338 |
// Square and right shift by 1 the odd element using 32 bit multiply |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8339 |
movl(raxReg, Address(x, tmp1, Address::times_4, 0)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8340 |
imulq(raxReg, raxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8341 |
shrq(raxReg, 1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8342 |
adcq(tmp5, 0); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8343 |
movq(Address(z, tmp4, Address::times_4, 0), raxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8344 |
incrementl(tmp1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8345 |
addl(tmp4, 2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8346 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8347 |
// Square and right shift by 1 the rest using 64 bit multiply |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8348 |
bind(L_first_loop); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8349 |
cmpptr(tmp1, xlen); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8350 |
jccb(Assembler::equal, L_first_loop_exit); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8351 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8352 |
// Square |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8353 |
movq(raxReg, Address(x, tmp1, Address::times_4, 0)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8354 |
rorq(raxReg, 32); // convert big-endian to little-endian |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8355 |
mulq(raxReg); // 64-bit multiply rax * rax -> rdx:rax |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8356 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8357 |
// Right shift by 1 and save carry |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8358 |
shrq(tmp5, 1); // rdx:rax:tmp5 = (tmp5:rdx:rax) >>> 1 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8359 |
rcrq(rdxReg, 1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8360 |
rcrq(raxReg, 1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8361 |
adcq(tmp5, 0); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8362 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8363 |
// Store result in z |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8364 |
movq(Address(z, tmp4, Address::times_4, 0), rdxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8365 |
movq(Address(z, tmp4, Address::times_4, 8), raxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8366 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8367 |
// Update indices for x and z |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8368 |
addl(tmp1, 2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8369 |
addl(tmp4, 4); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8370 |
jmp(L_first_loop); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8371 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8372 |
bind(L_first_loop_exit); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8373 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8374 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8375 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8376 |
/** |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8377 |
* Perform the following multiply add operation using BMI2 instructions |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8378 |
* carry:sum = sum + op1*op2 + carry |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8379 |
* op2 should be in rdx |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8380 |
* op2 is preserved, all other registers are modified |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8381 |
*/ |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8382 |
void MacroAssembler::multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry, Register tmp2) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8383 |
// assert op2 is rdx |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8384 |
mulxq(tmp2, op1, op1); // op1 * op2 -> tmp2:op1 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8385 |
addq(sum, carry); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8386 |
adcq(tmp2, 0); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8387 |
addq(sum, op1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8388 |
adcq(tmp2, 0); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8389 |
movq(carry, tmp2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8390 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8391 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8392 |
/** |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8393 |
* Perform the following multiply add operation: |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8394 |
* carry:sum = sum + op1*op2 + carry |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8395 |
* Preserves op1, op2 and modifies rest of registers |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8396 |
*/ |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8397 |
void MacroAssembler::multiply_add_64(Register sum, Register op1, Register op2, Register carry, Register rdxReg, Register raxReg) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8398 |
// rdx:rax = op1 * op2 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8399 |
movq(raxReg, op2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8400 |
mulq(op1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8401 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8402 |
// rdx:rax = sum + carry + rdx:rax |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8403 |
addq(sum, carry); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8404 |
adcq(rdxReg, 0); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8405 |
addq(sum, raxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8406 |
adcq(rdxReg, 0); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8407 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8408 |
// carry:sum = rdx:sum |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8409 |
movq(carry, rdxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8410 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8411 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8412 |
/** |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8413 |
* Add 64 bit long carry into z[] with carry propogation. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8414 |
* Preserves z and carry register values and modifies rest of registers. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8415 |
* |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8416 |
*/ |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8417 |
void MacroAssembler::add_one_64(Register z, Register zlen, Register carry, Register tmp1) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8418 |
Label L_fourth_loop, L_fourth_loop_exit; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8419 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8420 |
movl(tmp1, 1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8421 |
subl(zlen, 2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8422 |
addq(Address(z, zlen, Address::times_4, 0), carry); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8423 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8424 |
bind(L_fourth_loop); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8425 |
jccb(Assembler::carryClear, L_fourth_loop_exit); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8426 |
subl(zlen, 2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8427 |
jccb(Assembler::negative, L_fourth_loop_exit); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8428 |
addq(Address(z, zlen, Address::times_4, 0), tmp1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8429 |
jmp(L_fourth_loop); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8430 |
bind(L_fourth_loop_exit); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8431 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8432 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8433 |
/** |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8434 |
* Shift z[] left by 1 bit. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8435 |
* Preserves x, len, z and zlen registers and modifies rest of the registers. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8436 |
* |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8437 |
*/ |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8438 |
void MacroAssembler::lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8439 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8440 |
Label L_fifth_loop, L_fifth_loop_exit; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8441 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8442 |
// Fifth loop |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8443 |
// Perform primitiveLeftShift(z, zlen, 1) |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8444 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8445 |
const Register prev_carry = tmp1; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8446 |
const Register new_carry = tmp4; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8447 |
const Register value = tmp2; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8448 |
const Register zidx = tmp3; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8449 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8450 |
// int zidx, carry; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8451 |
// long value; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8452 |
// carry = 0; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8453 |
// for (zidx = zlen-2; zidx >=0; zidx -= 2) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8454 |
// (carry:value) = (z[i] << 1) | carry ; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8455 |
// z[i] = value; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8456 |
// } |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8457 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8458 |
movl(zidx, zlen); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8459 |
xorl(prev_carry, prev_carry); // clear carry flag and prev_carry register |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8460 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8461 |
bind(L_fifth_loop); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8462 |
decl(zidx); // Use decl to preserve carry flag |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8463 |
decl(zidx); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8464 |
jccb(Assembler::negative, L_fifth_loop_exit); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8465 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8466 |
if (UseBMI2Instructions) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8467 |
movq(value, Address(z, zidx, Address::times_4, 0)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8468 |
rclq(value, 1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8469 |
rorxq(value, value, 32); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8470 |
movq(Address(z, zidx, Address::times_4, 0), value); // Store back in big endian form |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8471 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8472 |
else { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8473 |
// clear new_carry |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8474 |
xorl(new_carry, new_carry); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8475 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8476 |
// Shift z[i] by 1, or in previous carry and save new carry |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8477 |
movq(value, Address(z, zidx, Address::times_4, 0)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8478 |
shlq(value, 1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8479 |
adcl(new_carry, 0); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8480 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8481 |
orq(value, prev_carry); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8482 |
rorq(value, 0x20); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8483 |
movq(Address(z, zidx, Address::times_4, 0), value); // Store back in big endian form |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8484 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8485 |
// Set previous carry = new carry |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8486 |
movl(prev_carry, new_carry); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8487 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8488 |
jmp(L_fifth_loop); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8489 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8490 |
bind(L_fifth_loop_exit); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8491 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8492 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8493 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8494 |
/** |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8495 |
* Code for BigInteger::squareToLen() intrinsic |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8496 |
* |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8497 |
* rdi: x |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8498 |
* rsi: len |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8499 |
* r8: z |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8500 |
* rcx: zlen |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8501 |
* r12: tmp1 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8502 |
* r13: tmp2 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8503 |
* r14: tmp3 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8504 |
* r15: tmp4 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8505 |
* rbx: tmp5 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8506 |
* |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8507 |
*/ |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8508 |
void MacroAssembler::square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8509 |
|
51756 | 8510 |
Label L_second_loop, L_second_loop_exit, L_third_loop, L_third_loop_exit, L_last_x, L_multiply; |
31129
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8511 |
push(tmp1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8512 |
push(tmp2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8513 |
push(tmp3); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8514 |
push(tmp4); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8515 |
push(tmp5); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8516 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8517 |
// First loop |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8518 |
// Store the squares, right shifted one bit (i.e., divided by 2). |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8519 |
square_rshift(x, len, z, tmp1, tmp3, tmp4, tmp5, rdxReg, raxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8520 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8521 |
// Add in off-diagonal sums. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8522 |
// |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8523 |
// Second, third (nested) and fourth loops. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8524 |
// zlen +=2; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8525 |
// for (int xidx=len-2,zidx=zlen-4; xidx > 0; xidx-=2,zidx-=4) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8526 |
// carry = 0; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8527 |
// long op2 = x[xidx:xidx+1]; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8528 |
// for (int j=xidx-2,k=zidx; j >= 0; j-=2) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8529 |
// k -= 2; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8530 |
// long op1 = x[j:j+1]; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8531 |
// long sum = z[k:k+1]; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8532 |
// carry:sum = multiply_add_64(sum, op1, op2, carry, tmp_regs); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8533 |
// z[k:k+1] = sum; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8534 |
// } |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8535 |
// add_one_64(z, k, carry, tmp_regs); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8536 |
// } |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8537 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8538 |
const Register carry = tmp5; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8539 |
const Register sum = tmp3; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8540 |
const Register op1 = tmp4; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8541 |
Register op2 = tmp2; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8542 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8543 |
push(zlen); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8544 |
push(len); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8545 |
addl(zlen,2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8546 |
bind(L_second_loop); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8547 |
xorq(carry, carry); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8548 |
subl(zlen, 4); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8549 |
subl(len, 2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8550 |
push(zlen); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8551 |
push(len); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8552 |
cmpl(len, 0); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8553 |
jccb(Assembler::lessEqual, L_second_loop_exit); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8554 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8555 |
// Multiply an array by one 64 bit long. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8556 |
if (UseBMI2Instructions) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8557 |
op2 = rdxReg; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8558 |
movq(op2, Address(x, len, Address::times_4, 0)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8559 |
rorxq(op2, op2, 32); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8560 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8561 |
else { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8562 |
movq(op2, Address(x, len, Address::times_4, 0)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8563 |
rorq(op2, 32); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8564 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8565 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8566 |
bind(L_third_loop); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8567 |
decrementl(len); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8568 |
jccb(Assembler::negative, L_third_loop_exit); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8569 |
decrementl(len); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8570 |
jccb(Assembler::negative, L_last_x); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8571 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8572 |
movq(op1, Address(x, len, Address::times_4, 0)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8573 |
rorq(op1, 32); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8574 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8575 |
bind(L_multiply); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8576 |
subl(zlen, 2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8577 |
movq(sum, Address(z, zlen, Address::times_4, 0)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8578 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8579 |
// Multiply 64 bit by 64 bit and add 64 bits lower half and upper 64 bits as carry. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8580 |
if (UseBMI2Instructions) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8581 |
multiply_add_64_bmi2(sum, op1, op2, carry, tmp2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8582 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8583 |
else { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8584 |
multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8585 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8586 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8587 |
movq(Address(z, zlen, Address::times_4, 0), sum); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8588 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8589 |
jmp(L_third_loop); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8590 |
bind(L_third_loop_exit); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8591 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8592 |
// Fourth loop |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8593 |
// Add 64 bit long carry into z with carry propogation. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8594 |
// Uses offsetted zlen. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8595 |
add_one_64(z, zlen, carry, tmp1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8596 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8597 |
pop(len); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8598 |
pop(zlen); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8599 |
jmp(L_second_loop); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8600 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8601 |
// Next infrequent code is moved outside loops. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8602 |
bind(L_last_x); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8603 |
movl(op1, Address(x, 0)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8604 |
jmp(L_multiply); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8605 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8606 |
bind(L_second_loop_exit); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8607 |
pop(len); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8608 |
pop(zlen); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8609 |
pop(len); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8610 |
pop(zlen); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8611 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8612 |
// Fifth loop |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8613 |
// Shift z left 1 bit. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8614 |
lshift_by_1(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8615 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8616 |
// z[zlen-1] |= x[len-1] & 1; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8617 |
movl(tmp3, Address(x, len, Address::times_4, -4)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8618 |
andl(tmp3, 1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8619 |
orl(Address(z, zlen, Address::times_4, -4), tmp3); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8620 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8621 |
pop(tmp5); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8622 |
pop(tmp4); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8623 |
pop(tmp3); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8624 |
pop(tmp2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8625 |
pop(tmp1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8626 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8627 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8628 |
/** |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8629 |
* Helper function for mul_add() |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8630 |
* Multiply the in[] by int k and add to out[] starting at offset offs using |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8631 |
* 128 bit by 32 bit multiply and return the carry in tmp5. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8632 |
* Only quad int aligned length of in[] is operated on in this function. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8633 |
* k is in rdxReg for BMI2Instructions, for others it is in tmp2. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8634 |
* This function preserves out, in and k registers. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8635 |
* len and offset point to the appropriate index in "in" & "out" correspondingly |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8636 |
* tmp5 has the carry. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8637 |
* other registers are temporary and are modified. |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8638 |
* |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8639 |
*/ |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8640 |
void MacroAssembler::mul_add_128_x_32_loop(Register out, Register in, |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8641 |
Register offset, Register len, Register tmp1, Register tmp2, Register tmp3, |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8642 |
Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8643 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8644 |
Label L_first_loop, L_first_loop_exit; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8645 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8646 |
movl(tmp1, len); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8647 |
shrl(tmp1, 2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8648 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8649 |
bind(L_first_loop); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8650 |
subl(tmp1, 1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8651 |
jccb(Assembler::negative, L_first_loop_exit); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8652 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8653 |
subl(len, 4); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8654 |
subl(offset, 4); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8655 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8656 |
Register op2 = tmp2; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8657 |
const Register sum = tmp3; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8658 |
const Register op1 = tmp4; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8659 |
const Register carry = tmp5; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8660 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8661 |
if (UseBMI2Instructions) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8662 |
op2 = rdxReg; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8663 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8664 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8665 |
movq(op1, Address(in, len, Address::times_4, 8)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8666 |
rorq(op1, 32); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8667 |
movq(sum, Address(out, offset, Address::times_4, 8)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8668 |
rorq(sum, 32); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8669 |
if (UseBMI2Instructions) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8670 |
multiply_add_64_bmi2(sum, op1, op2, carry, raxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8671 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8672 |
else { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8673 |
multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8674 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8675 |
// Store back in big endian from little endian |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8676 |
rorq(sum, 0x20); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8677 |
movq(Address(out, offset, Address::times_4, 8), sum); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8678 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8679 |
movq(op1, Address(in, len, Address::times_4, 0)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8680 |
rorq(op1, 32); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8681 |
movq(sum, Address(out, offset, Address::times_4, 0)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8682 |
rorq(sum, 32); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8683 |
if (UseBMI2Instructions) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8684 |
multiply_add_64_bmi2(sum, op1, op2, carry, raxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8685 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8686 |
else { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8687 |
multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8688 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8689 |
// Store back in big endian from little endian |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8690 |
rorq(sum, 0x20); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8691 |
movq(Address(out, offset, Address::times_4, 0), sum); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8692 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8693 |
jmp(L_first_loop); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8694 |
bind(L_first_loop_exit); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8695 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8696 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8697 |
/** |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8698 |
* Code for BigInteger::mulAdd() intrinsic |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8699 |
* |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8700 |
* rdi: out |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8701 |
* rsi: in |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8702 |
* r11: offs (out.length - offset) |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8703 |
* rcx: len |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8704 |
* r8: k |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8705 |
* r12: tmp1 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8706 |
* r13: tmp2 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8707 |
* r14: tmp3 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8708 |
* r15: tmp4 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8709 |
* rbx: tmp5 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8710 |
* Multiply the in[] by word k and add to out[], return the carry in rax |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8711 |
*/ |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8712 |
void MacroAssembler::mul_add(Register out, Register in, Register offs, |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8713 |
Register len, Register k, Register tmp1, Register tmp2, Register tmp3, |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8714 |
Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8715 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8716 |
Label L_carry, L_last_in, L_done; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8717 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8718 |
// carry = 0; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8719 |
// for (int j=len-1; j >= 0; j--) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8720 |
// long product = (in[j] & LONG_MASK) * kLong + |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8721 |
// (out[offs] & LONG_MASK) + carry; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8722 |
// out[offs--] = (int)product; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8723 |
// carry = product >>> 32; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8724 |
// } |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8725 |
// |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8726 |
push(tmp1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8727 |
push(tmp2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8728 |
push(tmp3); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8729 |
push(tmp4); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8730 |
push(tmp5); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8731 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8732 |
Register op2 = tmp2; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8733 |
const Register sum = tmp3; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8734 |
const Register op1 = tmp4; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8735 |
const Register carry = tmp5; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8736 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8737 |
if (UseBMI2Instructions) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8738 |
op2 = rdxReg; |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8739 |
movl(op2, k); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8740 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8741 |
else { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8742 |
movl(op2, k); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8743 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8744 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8745 |
xorq(carry, carry); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8746 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8747 |
//First loop |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8748 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8749 |
//Multiply in[] by k in a 4 way unrolled loop using 128 bit by 32 bit multiply |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8750 |
//The carry is in tmp5 |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8751 |
mul_add_128_x_32_loop(out, in, offs, len, tmp1, tmp2, tmp3, tmp4, tmp5, rdxReg, raxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8752 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8753 |
//Multiply the trailing in[] entry using 64 bit by 32 bit, if any |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8754 |
decrementl(len); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8755 |
jccb(Assembler::negative, L_carry); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8756 |
decrementl(len); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8757 |
jccb(Assembler::negative, L_last_in); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8758 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8759 |
movq(op1, Address(in, len, Address::times_4, 0)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8760 |
rorq(op1, 32); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8761 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8762 |
subl(offs, 2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8763 |
movq(sum, Address(out, offs, Address::times_4, 0)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8764 |
rorq(sum, 32); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8765 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8766 |
if (UseBMI2Instructions) { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8767 |
multiply_add_64_bmi2(sum, op1, op2, carry, raxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8768 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8769 |
else { |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8770 |
multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8771 |
} |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8772 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8773 |
// Store back in big endian from little endian |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8774 |
rorq(sum, 0x20); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8775 |
movq(Address(out, offs, Address::times_4, 0), sum); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8776 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8777 |
testl(len, len); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8778 |
jccb(Assembler::zero, L_carry); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8779 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8780 |
//Multiply the last in[] entry, if any |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8781 |
bind(L_last_in); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8782 |
movl(op1, Address(in, 0)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8783 |
movl(sum, Address(out, offs, Address::times_4, -4)); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8784 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8785 |
movl(raxReg, k); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8786 |
mull(op1); //tmp4 * eax -> edx:eax |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8787 |
addl(sum, carry); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8788 |
adcl(rdxReg, 0); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8789 |
addl(sum, raxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8790 |
adcl(rdxReg, 0); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8791 |
movl(carry, rdxReg); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8792 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8793 |
movl(Address(out, offs, Address::times_4, -4), sum); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8794 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8795 |
bind(L_carry); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8796 |
//return tmp5/carry as carry in rax |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8797 |
movl(rax, carry); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8798 |
|
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8799 |
bind(L_done); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8800 |
pop(tmp5); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8801 |
pop(tmp4); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8802 |
pop(tmp3); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8803 |
pop(tmp2); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8804 |
pop(tmp1); |
02ee7609f0e1
8081778: Use Intel x64 CPU instructions for RSA acceleration
kvn
parents:
30773
diff
changeset
|
8805 |
} |
26434
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8806 |
#endif |
09ad55e5f486
8055494: Add C2 x86 intrinsic for BigInteger::multiplyToLen() method
kvn
parents:
25932
diff
changeset
|
8807 |
|
18507
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8808 |
/** |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8809 |
* Emits code to update CRC-32 with a byte value according to constants in table |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8810 |
* |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8811 |
* @param [in,out]crc Register containing the crc. |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8812 |
* @param [in]val Register containing the byte to fold into the CRC. |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8813 |
* @param [in]table Register containing the table of crc constants. |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8814 |
* |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8815 |
* uint32_t crc; |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8816 |
* val = crc_table[(val ^ crc) & 0xFF]; |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8817 |
* crc = val ^ (crc >> 8); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8818 |
* |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8819 |
*/ |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8820 |
void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) { |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8821 |
xorl(val, crc); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8822 |
andl(val, 0xFF); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8823 |
shrl(crc, 8); // unsigned shift |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8824 |
xorl(crc, Address(table, val, Address::times_4, 0)); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8825 |
} |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8826 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8827 |
/** |
49614
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8828 |
* Fold four 128-bit data chunks |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8829 |
*/ |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8830 |
void MacroAssembler::fold_128bit_crc32_avx512(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) { |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8831 |
evpclmulhdq(xtmp, xK, xcrc, Assembler::AVX_512bit); // [123:64] |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8832 |
evpclmulldq(xcrc, xK, xcrc, Assembler::AVX_512bit); // [63:0] |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8833 |
evpxorq(xcrc, xcrc, Address(buf, offset), Assembler::AVX_512bit /* vector_len */); |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8834 |
evpxorq(xcrc, xcrc, xtmp, Assembler::AVX_512bit /* vector_len */); |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8835 |
} |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8836 |
|
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8837 |
/** |
18507
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8838 |
* Fold 128-bit data chunk |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8839 |
*/ |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8840 |
void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) { |
25932
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
8841 |
if (UseAVX > 0) { |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
8842 |
vpclmulhdq(xtmp, xK, xcrc); // [123:64] |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
8843 |
vpclmulldq(xcrc, xK, xcrc); // [63:0] |
30624 | 8844 |
vpxor(xcrc, xcrc, Address(buf, offset), 0 /* vector_len */); |
25932
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
8845 |
pxor(xcrc, xtmp); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
8846 |
} else { |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
8847 |
movdqa(xtmp, xcrc); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
8848 |
pclmulhdq(xtmp, xK); // [123:64] |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
8849 |
pclmulldq(xcrc, xK); // [63:0] |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
8850 |
pxor(xcrc, xtmp); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
8851 |
movdqu(xtmp, Address(buf, offset)); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
8852 |
pxor(xcrc, xtmp); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
8853 |
} |
18507
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8854 |
} |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8855 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8856 |
void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf) { |
25932
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
8857 |
if (UseAVX > 0) { |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
8858 |
vpclmulhdq(xtmp, xK, xcrc); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
8859 |
vpclmulldq(xcrc, xK, xcrc); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
8860 |
pxor(xcrc, xbuf); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
8861 |
pxor(xcrc, xtmp); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
8862 |
} else { |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
8863 |
movdqa(xtmp, xcrc); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
8864 |
pclmulhdq(xtmp, xK); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
8865 |
pclmulldq(xcrc, xK); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
8866 |
pxor(xcrc, xbuf); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
8867 |
pxor(xcrc, xtmp); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
8868 |
} |
18507
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8869 |
} |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8870 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8871 |
/** |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8872 |
* 8-bit folds to compute 32-bit CRC |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8873 |
* |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8874 |
* uint64_t xcrc; |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8875 |
* timesXtoThe32[xcrc & 0xFF] ^ (xcrc >> 8); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8876 |
*/ |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8877 |
void MacroAssembler::fold_8bit_crc32(XMMRegister xcrc, Register table, XMMRegister xtmp, Register tmp) { |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8878 |
movdl(tmp, xcrc); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8879 |
andl(tmp, 0xFF); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8880 |
movdl(xtmp, Address(table, tmp, Address::times_4, 0)); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8881 |
psrldq(xcrc, 1); // unsigned shift one byte |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8882 |
pxor(xcrc, xtmp); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8883 |
} |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8884 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8885 |
/** |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8886 |
* uint32_t crc; |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8887 |
* timesXtoThe32[crc & 0xFF] ^ (crc >> 8); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8888 |
*/ |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8889 |
void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) { |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8890 |
movl(tmp, crc); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8891 |
andl(tmp, 0xFF); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8892 |
shrl(crc, 8); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8893 |
xorl(crc, Address(table, tmp, Address::times_4, 0)); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8894 |
} |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8895 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8896 |
/** |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8897 |
* @param crc register containing existing CRC (32-bit) |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8898 |
* @param buf register pointing to input byte buffer (byte*) |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8899 |
* @param len register containing number of bytes |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8900 |
* @param table register that will contain address of CRC table |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8901 |
* @param tmp scratch register |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8902 |
*/ |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8903 |
void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp) { |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8904 |
assert_different_registers(crc, buf, len, table, tmp, rax); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8905 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8906 |
Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned; |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8907 |
Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop; |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8908 |
|
32727
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
8909 |
// For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
8910 |
// context for the registers used, where all instructions below are using 128-bit mode |
320855c2baef
8132160: support for AVX 512 call frames and stack management
mcberg
parents:
32599
diff
changeset
|
8911 |
// On EVEX without VL and BW, these instructions will all be AVX. |
18507
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8912 |
lea(table, ExternalAddress(StubRoutines::crc_table_addr())); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8913 |
notl(crc); // ~crc |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8914 |
cmpl(len, 16); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8915 |
jcc(Assembler::less, L_tail); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8916 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8917 |
// Align buffer to 16 bytes |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8918 |
movl(tmp, buf); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8919 |
andl(tmp, 0xF); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8920 |
jccb(Assembler::zero, L_aligned); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8921 |
subl(tmp, 16); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8922 |
addl(len, tmp); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8923 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8924 |
align(4); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8925 |
BIND(L_align_loop); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8926 |
movsbl(rax, Address(buf, 0)); // load byte with sign extension |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8927 |
update_byte_crc32(crc, rax, table); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8928 |
increment(buf); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8929 |
incrementl(tmp); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8930 |
jccb(Assembler::less, L_align_loop); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8931 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8932 |
BIND(L_aligned); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8933 |
movl(tmp, len); // save |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8934 |
shrl(len, 4); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8935 |
jcc(Assembler::zero, L_tail_restore); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8936 |
|
49614
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8937 |
// Fold total 512 bits of polynomial on each iteration |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8938 |
if (VM_Version::supports_vpclmulqdq()) { |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8939 |
Label Parallel_loop, L_No_Parallel; |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8940 |
|
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8941 |
cmpl(len, 8); |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8942 |
jccb(Assembler::less, L_No_Parallel); |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8943 |
|
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8944 |
movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32)); |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8945 |
evmovdquq(xmm1, Address(buf, 0), Assembler::AVX_512bit); |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8946 |
movdl(xmm5, crc); |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8947 |
evpxorq(xmm1, xmm1, xmm5, Assembler::AVX_512bit); |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8948 |
addptr(buf, 64); |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8949 |
subl(len, 7); |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8950 |
evshufi64x2(xmm0, xmm0, xmm0, 0x00, Assembler::AVX_512bit); //propagate the mask from 128 bits to 512 bits |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8951 |
|
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8952 |
BIND(Parallel_loop); |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8953 |
fold_128bit_crc32_avx512(xmm1, xmm0, xmm5, buf, 0); |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8954 |
addptr(buf, 64); |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8955 |
subl(len, 4); |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8956 |
jcc(Assembler::greater, Parallel_loop); |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8957 |
|
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8958 |
vextracti64x2(xmm2, xmm1, 0x01); |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8959 |
vextracti64x2(xmm3, xmm1, 0x02); |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8960 |
vextracti64x2(xmm4, xmm1, 0x03); |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8961 |
jmp(L_fold_512b); |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8962 |
|
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8963 |
BIND(L_No_Parallel); |
3b1570be8557
8200067: Add support for vpclmulqdq for crc32
srukmannagar
parents:
49455
diff
changeset
|
8964 |
} |
18507
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8965 |
// Fold crc into first bytes of vector |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8966 |
movdqa(xmm1, Address(buf, 0)); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8967 |
movdl(rax, xmm1); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8968 |
xorl(crc, rax); |
39256
ac12f57c6d9c
8158214: Crash with "assert(VM_Version::supports_sse4_1()) failed" if UseSSE < 4 is set
thartmann
parents:
39253
diff
changeset
|
8969 |
if (VM_Version::supports_sse4_1()) { |
ac12f57c6d9c
8158214: Crash with "assert(VM_Version::supports_sse4_1()) failed" if UseSSE < 4 is set
thartmann
parents:
39253
diff
changeset
|
8970 |
pinsrd(xmm1, crc, 0); |
ac12f57c6d9c
8158214: Crash with "assert(VM_Version::supports_sse4_1()) failed" if UseSSE < 4 is set
thartmann
parents:
39253
diff
changeset
|
8971 |
} else { |
ac12f57c6d9c
8158214: Crash with "assert(VM_Version::supports_sse4_1()) failed" if UseSSE < 4 is set
thartmann
parents:
39253
diff
changeset
|
8972 |
pinsrw(xmm1, crc, 0); |
ac12f57c6d9c
8158214: Crash with "assert(VM_Version::supports_sse4_1()) failed" if UseSSE < 4 is set
thartmann
parents:
39253
diff
changeset
|
8973 |
shrl(crc, 16); |
ac12f57c6d9c
8158214: Crash with "assert(VM_Version::supports_sse4_1()) failed" if UseSSE < 4 is set
thartmann
parents:
39253
diff
changeset
|
8974 |
pinsrw(xmm1, crc, 1); |
ac12f57c6d9c
8158214: Crash with "assert(VM_Version::supports_sse4_1()) failed" if UseSSE < 4 is set
thartmann
parents:
39253
diff
changeset
|
8975 |
} |
18507
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8976 |
addptr(buf, 16); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8977 |
subl(len, 4); // len > 0 |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8978 |
jcc(Assembler::less, L_fold_tail); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8979 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8980 |
movdqa(xmm2, Address(buf, 0)); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8981 |
movdqa(xmm3, Address(buf, 16)); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8982 |
movdqa(xmm4, Address(buf, 32)); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8983 |
addptr(buf, 48); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8984 |
subl(len, 3); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8985 |
jcc(Assembler::lessEqual, L_fold_512b); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8986 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8987 |
// Fold total 512 bits of polynomial on each iteration, |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8988 |
// 128 bits per each of 4 parallel streams. |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8989 |
movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32)); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8990 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8991 |
align(32); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8992 |
BIND(L_fold_512b_loop); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8993 |
fold_128bit_crc32(xmm1, xmm0, xmm5, buf, 0); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8994 |
fold_128bit_crc32(xmm2, xmm0, xmm5, buf, 16); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8995 |
fold_128bit_crc32(xmm3, xmm0, xmm5, buf, 32); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8996 |
fold_128bit_crc32(xmm4, xmm0, xmm5, buf, 48); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8997 |
addptr(buf, 64); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8998 |
subl(len, 4); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
8999 |
jcc(Assembler::greater, L_fold_512b_loop); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9000 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9001 |
// Fold 512 bits to 128 bits. |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9002 |
BIND(L_fold_512b); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9003 |
movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16)); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9004 |
fold_128bit_crc32(xmm1, xmm0, xmm5, xmm2); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9005 |
fold_128bit_crc32(xmm1, xmm0, xmm5, xmm3); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9006 |
fold_128bit_crc32(xmm1, xmm0, xmm5, xmm4); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9007 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9008 |
// Fold the rest of 128 bits data chunks |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9009 |
BIND(L_fold_tail); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9010 |
addl(len, 3); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9011 |
jccb(Assembler::lessEqual, L_fold_128b); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9012 |
movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16)); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9013 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9014 |
BIND(L_fold_tail_loop); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9015 |
fold_128bit_crc32(xmm1, xmm0, xmm5, buf, 0); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9016 |
addptr(buf, 16); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9017 |
decrementl(len); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9018 |
jccb(Assembler::greater, L_fold_tail_loop); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9019 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9020 |
// Fold 128 bits in xmm1 down into 32 bits in crc register. |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9021 |
BIND(L_fold_128b); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9022 |
movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr())); |
25932
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
9023 |
if (UseAVX > 0) { |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
9024 |
vpclmulqdq(xmm2, xmm0, xmm1, 0x1); |
30624 | 9025 |
vpand(xmm3, xmm0, xmm2, 0 /* vector_len */); |
25932
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
9026 |
vpclmulqdq(xmm0, xmm0, xmm3, 0x1); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
9027 |
} else { |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
9028 |
movdqa(xmm2, xmm0); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
9029 |
pclmulqdq(xmm2, xmm1, 0x1); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
9030 |
movdqa(xmm3, xmm0); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
9031 |
pand(xmm3, xmm2); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
9032 |
pclmulqdq(xmm0, xmm3, 0x1); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
24424
diff
changeset
|
9033 |
} |
18507
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9034 |
psrldq(xmm1, 8); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9035 |
psrldq(xmm2, 4); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9036 |
pxor(xmm0, xmm1); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9037 |
pxor(xmm0, xmm2); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9038 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9039 |
// 8 8-bit folds to compute 32-bit CRC. |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9040 |
for (int j = 0; j < 4; j++) { |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9041 |
fold_8bit_crc32(xmm0, table, xmm1, rax); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9042 |
} |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9043 |
movdl(crc, xmm0); // mov 32 bits to general register |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9044 |
for (int j = 0; j < 4; j++) { |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9045 |
fold_8bit_crc32(crc, table, rax); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9046 |
} |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9047 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9048 |
BIND(L_tail_restore); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9049 |
movl(len, tmp); // restore |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9050 |
BIND(L_tail); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9051 |
andl(len, 0xf); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9052 |
jccb(Assembler::zero, L_exit); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9053 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9054 |
// Fold the rest of bytes |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9055 |
align(4); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9056 |
BIND(L_tail_loop); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9057 |
movsbl(rax, Address(buf, 0)); // load byte with sign extension |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9058 |
update_byte_crc32(crc, rax, table); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9059 |
increment(buf); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9060 |
decrementl(len); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9061 |
jccb(Assembler::greater, L_tail_loop); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9062 |
|
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9063 |
BIND(L_exit); |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9064 |
notl(crc); // ~c |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9065 |
} |
61bfc8995bb3
7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32
drchase
parents:
16624
diff
changeset
|
9066 |
|
33066 | 9067 |
#ifdef _LP64 |
9068 |
// S. Gueron / Information Processing Letters 112 (2012) 184 |
|
9069 |
// Algorithm 4: Computing carry-less multiplication using a precomputed lookup table. |
|
9070 |
// Input: A 32 bit value B = [byte3, byte2, byte1, byte0]. |
|
9071 |
// Output: the 64-bit carry-less product of B * CONST |
|
9072 |
void MacroAssembler::crc32c_ipl_alg4(Register in, uint32_t n, |
|
9073 |
Register tmp1, Register tmp2, Register tmp3) { |
|
9074 |
lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr())); |
|
9075 |
if (n > 0) { |
|
9076 |
addq(tmp3, n * 256 * 8); |
|
9077 |
} |
|
9078 |
// Q1 = TABLEExt[n][B & 0xFF]; |
|
9079 |
movl(tmp1, in); |
|
9080 |
andl(tmp1, 0x000000FF); |
|
9081 |
shll(tmp1, 3); |
|
9082 |
addq(tmp1, tmp3); |
|
9083 |
movq(tmp1, Address(tmp1, 0)); |
|
9084 |
||
9085 |
// Q2 = TABLEExt[n][B >> 8 & 0xFF]; |
|
9086 |
movl(tmp2, in); |
|
9087 |
shrl(tmp2, 8); |
|
9088 |
andl(tmp2, 0x000000FF); |
|
9089 |
shll(tmp2, 3); |
|
9090 |
addq(tmp2, tmp3); |
|
9091 |
movq(tmp2, Address(tmp2, 0)); |
|
9092 |
||
9093 |
shlq(tmp2, 8); |
|
9094 |
xorq(tmp1, tmp2); |
|
9095 |
||
9096 |
// Q3 = TABLEExt[n][B >> 16 & 0xFF]; |
|
9097 |
movl(tmp2, in); |
|
9098 |
shrl(tmp2, 16); |
|
9099 |
andl(tmp2, 0x000000FF); |
|
9100 |
shll(tmp2, 3); |
|
9101 |
addq(tmp2, tmp3); |
|
9102 |
movq(tmp2, Address(tmp2, 0)); |
|
9103 |
||
9104 |
shlq(tmp2, 16); |
|
9105 |
xorq(tmp1, tmp2); |
|
9106 |
||
9107 |
// Q4 = TABLEExt[n][B >> 24 & 0xFF]; |
|
9108 |
shrl(in, 24); |
|
9109 |
andl(in, 0x000000FF); |
|
9110 |
shll(in, 3); |
|
9111 |
addq(in, tmp3); |
|
9112 |
movq(in, Address(in, 0)); |
|
9113 |
||
9114 |
shlq(in, 24); |
|
9115 |
xorq(in, tmp1); |
|
9116 |
// return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24; |
|
9117 |
} |
|
9118 |
||
9119 |
void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1, |
|
9120 |
Register in_out, |
|
9121 |
uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported, |
|
9122 |
XMMRegister w_xtmp2, |
|
9123 |
Register tmp1, |
|
9124 |
Register n_tmp2, Register n_tmp3) { |
|
9125 |
if (is_pclmulqdq_supported) { |
|
9126 |
movdl(w_xtmp1, in_out); // modified blindly |
|
9127 |
||
9128 |
movl(tmp1, const_or_pre_comp_const_index); |
|
9129 |
movdl(w_xtmp2, tmp1); |
|
9130 |
pclmulqdq(w_xtmp1, w_xtmp2, 0); |
|
9131 |
||
9132 |
movdq(in_out, w_xtmp1); |
|
9133 |
} else { |
|
9134 |
crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3); |
|
9135 |
} |
|
9136 |
} |
|
9137 |
||
9138 |
// Recombination Alternative 2: No bit-reflections |
|
9139 |
// T1 = (CRC_A * U1) << 1 |
|
9140 |
// T2 = (CRC_B * U2) << 1 |
|
9141 |
// C1 = T1 >> 32 |
|
9142 |
// C2 = T2 >> 32 |
|
9143 |
// T1 = T1 & 0xFFFFFFFF |
|
9144 |
// T2 = T2 & 0xFFFFFFFF |
|
9145 |
// T1 = CRC32(0, T1) |
|
9146 |
// T2 = CRC32(0, T2) |
|
9147 |
// C1 = C1 ^ T1 |
|
9148 |
// C2 = C2 ^ T2 |
|
9149 |
// CRC = C1 ^ C2 ^ CRC_C |
|
9150 |
void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2, |
|
9151 |
XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, |
|
9152 |
Register tmp1, Register tmp2, |
|
9153 |
Register n_tmp3) { |
|
9154 |
crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); |
|
9155 |
crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); |
|
9156 |
shlq(in_out, 1); |
|
9157 |
movl(tmp1, in_out); |
|
9158 |
shrq(in_out, 32); |
|
9159 |
xorl(tmp2, tmp2); |
|
9160 |
crc32(tmp2, tmp1, 4); |
|
9161 |
xorl(in_out, tmp2); // we don't care about upper 32 bit contents here |
|
9162 |
shlq(in1, 1); |
|
9163 |
movl(tmp1, in1); |
|
9164 |
shrq(in1, 32); |
|
9165 |
xorl(tmp2, tmp2); |
|
9166 |
crc32(tmp2, tmp1, 4); |
|
9167 |
xorl(in1, tmp2); |
|
9168 |
xorl(in_out, in1); |
|
9169 |
xorl(in_out, in2); |
|
9170 |
} |
|
9171 |
||
9172 |
// Set N to predefined value |
|
9173 |
// Subtract from a lenght of a buffer |
|
9174 |
// execute in a loop: |
|
9175 |
// CRC_A = 0xFFFFFFFF, CRC_B = 0, CRC_C = 0 |
|
9176 |
// for i = 1 to N do |
|
9177 |
// CRC_A = CRC32(CRC_A, A[i]) |
|
9178 |
// CRC_B = CRC32(CRC_B, B[i]) |
|
9179 |
// CRC_C = CRC32(CRC_C, C[i]) |
|
9180 |
// end for |
|
9181 |
// Recombine |
|
9182 |
void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, |
|
9183 |
Register in_out1, Register in_out2, Register in_out3, |
|
9184 |
Register tmp1, Register tmp2, Register tmp3, |
|
9185 |
XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, |
|
9186 |
Register tmp4, Register tmp5, |
|
9187 |
Register n_tmp6) { |
|
9188 |
Label L_processPartitions; |
|
9189 |
Label L_processPartition; |
|
9190 |
Label L_exit; |
|
9191 |
||
9192 |
bind(L_processPartitions); |
|
9193 |
cmpl(in_out1, 3 * size); |
|
9194 |
jcc(Assembler::less, L_exit); |
|
9195 |
xorl(tmp1, tmp1); |
|
9196 |
xorl(tmp2, tmp2); |
|
9197 |
movq(tmp3, in_out2); |
|
9198 |
addq(tmp3, size); |
|
9199 |
||
9200 |
bind(L_processPartition); |
|
9201 |
crc32(in_out3, Address(in_out2, 0), 8); |
|
9202 |
crc32(tmp1, Address(in_out2, size), 8); |
|
9203 |
crc32(tmp2, Address(in_out2, size * 2), 8); |
|
9204 |
addq(in_out2, 8); |
|
9205 |
cmpq(in_out2, tmp3); |
|
9206 |
jcc(Assembler::less, L_processPartition); |
|
9207 |
crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2, |
|
9208 |
w_xtmp1, w_xtmp2, w_xtmp3, |
|
9209 |
tmp4, tmp5, |
|
9210 |
n_tmp6); |
|
9211 |
addq(in_out2, 2 * size); |
|
9212 |
subl(in_out1, 3 * size); |
|
9213 |
jmp(L_processPartitions); |
|
9214 |
||
9215 |
bind(L_exit); |
|
9216 |
} |
|
9217 |
#else |
|
9218 |
void MacroAssembler::crc32c_ipl_alg4(Register in_out, uint32_t n, |
|
9219 |
Register tmp1, Register tmp2, Register tmp3, |
|
9220 |
XMMRegister xtmp1, XMMRegister xtmp2) { |
|
9221 |
lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr())); |
|
9222 |
if (n > 0) { |
|
9223 |
addl(tmp3, n * 256 * 8); |
|
9224 |
} |
|
9225 |
// Q1 = TABLEExt[n][B & 0xFF]; |
|
9226 |
movl(tmp1, in_out); |
|
9227 |
andl(tmp1, 0x000000FF); |
|
9228 |
shll(tmp1, 3); |
|
9229 |
addl(tmp1, tmp3); |
|
9230 |
movq(xtmp1, Address(tmp1, 0)); |
|
9231 |
||
9232 |
// Q2 = TABLEExt[n][B >> 8 & 0xFF]; |
|
9233 |
movl(tmp2, in_out); |
|
9234 |
shrl(tmp2, 8); |
|
9235 |
andl(tmp2, 0x000000FF); |
|
9236 |
shll(tmp2, 3); |
|
9237 |
addl(tmp2, tmp3); |
|
9238 |
movq(xtmp2, Address(tmp2, 0)); |
|
9239 |
||
9240 |
psllq(xtmp2, 8); |
|
9241 |
pxor(xtmp1, xtmp2); |
|
9242 |
||
9243 |
// Q3 = TABLEExt[n][B >> 16 & 0xFF]; |
|
9244 |
movl(tmp2, in_out); |
|
9245 |
shrl(tmp2, 16); |
|
9246 |
andl(tmp2, 0x000000FF); |
|
9247 |
shll(tmp2, 3); |
|
9248 |
addl(tmp2, tmp3); |
|
9249 |
movq(xtmp2, Address(tmp2, 0)); |
|
9250 |
||
9251 |
psllq(xtmp2, 16); |
|
9252 |
pxor(xtmp1, xtmp2); |
|
9253 |
||
9254 |
// Q4 = TABLEExt[n][B >> 24 & 0xFF]; |
|
9255 |
shrl(in_out, 24); |
|
9256 |
andl(in_out, 0x000000FF); |
|
9257 |
shll(in_out, 3); |
|
9258 |
addl(in_out, tmp3); |
|
9259 |
movq(xtmp2, Address(in_out, 0)); |
|
9260 |
||
9261 |
psllq(xtmp2, 24); |
|
9262 |
pxor(xtmp1, xtmp2); // Result in CXMM |
|
9263 |
// return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24; |
|
9264 |
} |
|
9265 |
||
9266 |
void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1, |
|
9267 |
Register in_out, |
|
9268 |
uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported, |
|
9269 |
XMMRegister w_xtmp2, |
|
9270 |
Register tmp1, |
|
9271 |
Register n_tmp2, Register n_tmp3) { |
|
9272 |
if (is_pclmulqdq_supported) { |
|
9273 |
movdl(w_xtmp1, in_out); |
|
9274 |
||
9275 |
movl(tmp1, const_or_pre_comp_const_index); |
|
9276 |
movdl(w_xtmp2, tmp1); |
|
9277 |
pclmulqdq(w_xtmp1, w_xtmp2, 0); |
|
9278 |
// Keep result in XMM since GPR is 32 bit in length |
|
9279 |
} else { |
|
9280 |
crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3, w_xtmp1, w_xtmp2); |
|
9281 |
} |
|
9282 |
} |
|
9283 |
||
9284 |
void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2, |
|
9285 |
XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, |
|
9286 |
Register tmp1, Register tmp2, |
|
9287 |
Register n_tmp3) { |
|
9288 |
crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); |
|
9289 |
crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); |
|
9290 |
||
9291 |
psllq(w_xtmp1, 1); |
|
9292 |
movdl(tmp1, w_xtmp1); |
|
9293 |
psrlq(w_xtmp1, 32); |
|
9294 |
movdl(in_out, w_xtmp1); |
|
9295 |
||
9296 |
xorl(tmp2, tmp2); |
|
9297 |
crc32(tmp2, tmp1, 4); |
|
9298 |
xorl(in_out, tmp2); |
|
9299 |
||
9300 |
psllq(w_xtmp2, 1); |
|
9301 |
movdl(tmp1, w_xtmp2); |
|
9302 |
psrlq(w_xtmp2, 32); |
|
9303 |
movdl(in1, w_xtmp2); |
|
9304 |
||
9305 |
xorl(tmp2, tmp2); |
|
9306 |
crc32(tmp2, tmp1, 4); |
|
9307 |
xorl(in1, tmp2); |
|
9308 |
xorl(in_out, in1); |
|
9309 |
xorl(in_out, in2); |
|
9310 |
} |
|
9311 |
||
9312 |
void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, |
|
9313 |
Register in_out1, Register in_out2, Register in_out3, |
|
9314 |
Register tmp1, Register tmp2, Register tmp3, |
|
9315 |
XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, |
|
9316 |
Register tmp4, Register tmp5, |
|
9317 |
Register n_tmp6) { |
|
9318 |
Label L_processPartitions; |
|
9319 |
Label L_processPartition; |
|
9320 |
Label L_exit; |
|
9321 |
||
9322 |
bind(L_processPartitions); |
|
9323 |
cmpl(in_out1, 3 * size); |
|
9324 |
jcc(Assembler::less, L_exit); |
|
9325 |
xorl(tmp1, tmp1); |
|
9326 |
xorl(tmp2, tmp2); |
|
9327 |
movl(tmp3, in_out2); |
|
9328 |
addl(tmp3, size); |
|
9329 |
||
9330 |
bind(L_processPartition); |
|
9331 |
crc32(in_out3, Address(in_out2, 0), 4); |
|
9332 |
crc32(tmp1, Address(in_out2, size), 4); |
|
9333 |
crc32(tmp2, Address(in_out2, size*2), 4); |
|
9334 |
crc32(in_out3, Address(in_out2, 0+4), 4); |
|
9335 |
crc32(tmp1, Address(in_out2, size+4), 4); |
|
9336 |
crc32(tmp2, Address(in_out2, size*2+4), 4); |
|
9337 |
addl(in_out2, 8); |
|
9338 |
cmpl(in_out2, tmp3); |
|
9339 |
jcc(Assembler::less, L_processPartition); |
|
9340 |
||
9341 |
push(tmp3); |
|
9342 |
push(in_out1); |
|
9343 |
push(in_out2); |
|
9344 |
tmp4 = tmp3; |
|
9345 |
tmp5 = in_out1; |
|
9346 |
n_tmp6 = in_out2; |
|
9347 |
||
9348 |
crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2, |
|
9349 |
w_xtmp1, w_xtmp2, w_xtmp3, |
|
9350 |
tmp4, tmp5, |
|
9351 |
n_tmp6); |
|
9352 |
||
9353 |
pop(in_out2); |
|
9354 |
pop(in_out1); |
|
9355 |
pop(tmp3); |
|
9356 |
||
9357 |
addl(in_out2, 2 * size); |
|
9358 |
subl(in_out1, 3 * size); |
|
9359 |
jmp(L_processPartitions); |
|
9360 |
||
9361 |
bind(L_exit); |
|
9362 |
} |
|
9363 |
#endif //LP64 |
|
9364 |
||
9365 |
#ifdef _LP64 |
|
9366 |
// Algorithm 2: Pipelined usage of the CRC32 instruction. |
|
9367 |
// Input: A buffer I of L bytes. |
|
9368 |
// Output: the CRC32C value of the buffer. |
|
9369 |
// Notations: |
|
9370 |
// Write L = 24N + r, with N = floor (L/24). |
|
9371 |
// r = L mod 24 (0 <= r < 24). |
|
9372 |
// Consider I as the concatenation of A|B|C|R, where A, B, C, each, |
|
9373 |
// N quadwords, and R consists of r bytes. |
|
9374 |
// A[j] = I [8j+7:8j], j= 0, 1, ..., N-1 |
|
9375 |
// B[j] = I [N + 8j+7:N + 8j], j= 0, 1, ..., N-1 |
|
9376 |
// C[j] = I [2N + 8j+7:2N + 8j], j= 0, 1, ..., N-1 |
|
9377 |
// if r > 0 R[j] = I [3N +j], j= 0, 1, ...,r-1 |
|
9378 |
void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2, |
|
9379 |
Register tmp1, Register tmp2, Register tmp3, |
|
9380 |
Register tmp4, Register tmp5, Register tmp6, |
|
9381 |
XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, |
|
9382 |
bool is_pclmulqdq_supported) { |
|
9383 |
uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS]; |
|
9384 |
Label L_wordByWord; |
|
9385 |
Label L_byteByByteProlog; |
|
9386 |
Label L_byteByByte; |
|
9387 |
Label L_exit; |
|
9388 |
||
9389 |
if (is_pclmulqdq_supported ) { |
|
9390 |
const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr; |
|
9391 |
const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr+1); |
|
9392 |
||
9393 |
const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2); |
|
9394 |
const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3); |
|
9395 |
||
9396 |
const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4); |
|
9397 |
const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5); |
|
9398 |
assert((CRC32C_NUM_PRECOMPUTED_CONSTANTS - 1 ) == 5, "Checking whether you declared all of the constants based on the number of \"chunks\""); |
|
9399 |
} else { |
|
9400 |
const_or_pre_comp_const_index[0] = 1; |
|
9401 |
const_or_pre_comp_const_index[1] = 0; |
|
9402 |
||
9403 |
const_or_pre_comp_const_index[2] = 3; |
|
9404 |
const_or_pre_comp_const_index[3] = 2; |
|
9405 |
||
9406 |
const_or_pre_comp_const_index[4] = 5; |
|
9407 |
const_or_pre_comp_const_index[5] = 4; |
|
9408 |
} |
|
9409 |
crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported, |
|
9410 |
in2, in1, in_out, |
|
9411 |
tmp1, tmp2, tmp3, |
|
9412 |
w_xtmp1, w_xtmp2, w_xtmp3, |
|
9413 |
tmp4, tmp5, |
|
9414 |
tmp6); |
|
9415 |
crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported, |
|
9416 |
in2, in1, in_out, |
|
9417 |
tmp1, tmp2, tmp3, |
|
9418 |
w_xtmp1, w_xtmp2, w_xtmp3, |
|
9419 |
tmp4, tmp5, |
|
9420 |
tmp6); |
|
9421 |
crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported, |
|
9422 |
in2, in1, in_out, |
|
9423 |
tmp1, tmp2, tmp3, |
|
9424 |
w_xtmp1, w_xtmp2, w_xtmp3, |
|
9425 |
tmp4, tmp5, |
|
9426 |
tmp6); |
|
9427 |
movl(tmp1, in2); |
|
9428 |
andl(tmp1, 0x00000007); |
|
9429 |
negl(tmp1); |
|
9430 |
addl(tmp1, in2); |
|
9431 |
addq(tmp1, in1); |
|
9432 |
||
9433 |
BIND(L_wordByWord); |
|
9434 |
cmpq(in1, tmp1); |
|
9435 |
jcc(Assembler::greaterEqual, L_byteByByteProlog); |
|
9436 |
crc32(in_out, Address(in1, 0), 4); |
|
9437 |
addq(in1, 4); |
|
9438 |
jmp(L_wordByWord); |
|
9439 |
||
9440 |
BIND(L_byteByByteProlog); |
|
9441 |
andl(in2, 0x00000007); |
|
9442 |
movl(tmp2, 1); |
|
9443 |
||
9444 |
BIND(L_byteByByte); |
|
9445 |
cmpl(tmp2, in2); |
|
9446 |
jccb(Assembler::greater, L_exit); |
|
9447 |
crc32(in_out, Address(in1, 0), 1); |
|
9448 |
incq(in1); |
|
9449 |
incl(tmp2); |
|
9450 |
jmp(L_byteByByte); |
|
9451 |
||
9452 |
BIND(L_exit); |
|
9453 |
} |
|
9454 |
#else |
|
9455 |
void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2, |
|
9456 |
Register tmp1, Register tmp2, Register tmp3, |
|
9457 |
Register tmp4, Register tmp5, Register tmp6, |
|
9458 |
XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, |
|
9459 |
bool is_pclmulqdq_supported) { |
|
9460 |
uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS]; |
|
9461 |
Label L_wordByWord; |
|
9462 |
Label L_byteByByteProlog; |
|
9463 |
Label L_byteByByte; |
|
9464 |
Label L_exit; |
|
9465 |
||
9466 |
if (is_pclmulqdq_supported) { |
|
9467 |
const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr; |
|
9468 |
const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 1); |
|
9469 |
||
9470 |
const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2); |
|
9471 |
const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3); |
|
9472 |
||
9473 |
const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4); |
|
9474 |
const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5); |
|
9475 |
} else { |
|
9476 |
const_or_pre_comp_const_index[0] = 1; |
|
9477 |
const_or_pre_comp_const_index[1] = 0; |
|
9478 |
||
9479 |
const_or_pre_comp_const_index[2] = 3; |
|
9480 |
const_or_pre_comp_const_index[3] = 2; |
|
9481 |
||
9482 |
const_or_pre_comp_const_index[4] = 5; |
|
9483 |
const_or_pre_comp_const_index[5] = 4; |
|
9484 |
} |
|
9485 |
crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported, |
|
9486 |
in2, in1, in_out, |
|
9487 |
tmp1, tmp2, tmp3, |
|
9488 |
w_xtmp1, w_xtmp2, w_xtmp3, |
|
9489 |
tmp4, tmp5, |
|
9490 |
tmp6); |
|
9491 |
crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported, |
|
9492 |
in2, in1, in_out, |
|
9493 |
tmp1, tmp2, tmp3, |
|
9494 |
w_xtmp1, w_xtmp2, w_xtmp3, |
|
9495 |
tmp4, tmp5, |
|
9496 |
tmp6); |
|
9497 |
crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported, |
|
9498 |
in2, in1, in_out, |
|
9499 |
tmp1, tmp2, tmp3, |
|
9500 |
w_xtmp1, w_xtmp2, w_xtmp3, |
|
9501 |
tmp4, tmp5, |
|
9502 |
tmp6); |
|
9503 |
movl(tmp1, in2); |
|
9504 |
andl(tmp1, 0x00000007); |
|
9505 |
negl(tmp1); |
|
9506 |
addl(tmp1, in2); |
|
9507 |
addl(tmp1, in1); |
|
9508 |
||
9509 |
BIND(L_wordByWord); |
|
9510 |
cmpl(in1, tmp1); |
|
9511 |
jcc(Assembler::greaterEqual, L_byteByByteProlog); |
|
9512 |
crc32(in_out, Address(in1,0), 4); |
|
9513 |
addl(in1, 4); |
|
9514 |
jmp(L_wordByWord); |
|
9515 |
||
9516 |
BIND(L_byteByByteProlog); |
|
9517 |
andl(in2, 0x00000007); |
|
9518 |
movl(tmp2, 1); |
|
9519 |
||
9520 |
BIND(L_byteByByte); |
|
9521 |
cmpl(tmp2, in2); |
|
9522 |
jccb(Assembler::greater, L_exit); |
|
9523 |
movb(tmp1, Address(in1, 0)); |
|
9524 |
crc32(in_out, tmp1, 1); |
|
9525 |
incl(in1); |
|
9526 |
incl(tmp2); |
|
9527 |
jmp(L_byteByByte); |
|
9528 |
||
9529 |
BIND(L_exit); |
|
9530 |
} |
|
9531 |
#endif // LP64 |
|
14626 | 9532 |
#undef BIND |
9533 |
#undef BLOCK_COMMENT |
|
9534 |
||
33628 | 9535 |
// Compress char[] array to byte[]. |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9536 |
// ..\jdk\src\java.base\share\classes\java\lang\StringUTF16.java |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9537 |
// @HotSpotIntrinsicCandidate |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9538 |
// private static int compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) { |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9539 |
// for (int i = 0; i < len; i++) { |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9540 |
// int c = src[srcOff++]; |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9541 |
// if (c >>> 8 != 0) { |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9542 |
// return 0; |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9543 |
// } |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9544 |
// dst[dstOff++] = (byte)c; |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9545 |
// } |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9546 |
// return len; |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9547 |
// } |
33628 | 9548 |
void MacroAssembler::char_array_compress(Register src, Register dst, Register len, |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9549 |
XMMRegister tmp1Reg, XMMRegister tmp2Reg, |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9550 |
XMMRegister tmp3Reg, XMMRegister tmp4Reg, |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9551 |
Register tmp5, Register result) { |
51419
c11be049acb8
8209511: C2 asserts with UseSSE < 4 and AVX enabled: "Label was never bound to a location, but it was used as a jmp target'
thartmann
parents:
51350
diff
changeset
|
9552 |
Label copy_chars_loop, return_length, return_zero, done; |
33628 | 9553 |
|
9554 |
// rsi: src |
|
9555 |
// rdi: dst |
|
9556 |
// rdx: len |
|
9557 |
// rcx: tmp5 |
|
9558 |
// rax: result |
|
9559 |
||
9560 |
// rsi holds start addr of source char[] to be compressed |
|
9561 |
// rdi holds start addr of destination byte[] |
|
9562 |
// rdx holds length |
|
9563 |
||
9564 |
assert(len != result, ""); |
|
9565 |
||
9566 |
// save length for return |
|
9567 |
push(len); |
|
9568 |
||
58462
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
9569 |
if ((AVX3Threshold == 0) && (UseAVX > 2) && // AVX512 |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9570 |
VM_Version::supports_avx512vlbw() && |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9571 |
VM_Version::supports_bmi2()) { |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9572 |
|
51976
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
9573 |
Label copy_32_loop, copy_loop_tail, below_threshold; |
51419
c11be049acb8
8209511: C2 asserts with UseSSE < 4 and AVX enabled: "Label was never bound to a location, but it was used as a jmp target'
thartmann
parents:
51350
diff
changeset
|
9574 |
|
c11be049acb8
8209511: C2 asserts with UseSSE < 4 and AVX enabled: "Label was never bound to a location, but it was used as a jmp target'
thartmann
parents:
51350
diff
changeset
|
9575 |
// alignment |
c11be049acb8
8209511: C2 asserts with UseSSE < 4 and AVX enabled: "Label was never bound to a location, but it was used as a jmp target'
thartmann
parents:
51350
diff
changeset
|
9576 |
Label post_alignment; |
c11be049acb8
8209511: C2 asserts with UseSSE < 4 and AVX enabled: "Label was never bound to a location, but it was used as a jmp target'
thartmann
parents:
51350
diff
changeset
|
9577 |
|
c11be049acb8
8209511: C2 asserts with UseSSE < 4 and AVX enabled: "Label was never bound to a location, but it was used as a jmp target'
thartmann
parents:
51350
diff
changeset
|
9578 |
// if length of the string is less than 16, handle it in an old fashioned way |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9579 |
testl(len, -32); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9580 |
jcc(Assembler::zero, below_threshold); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9581 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9582 |
// First check whether a character is compressable ( <= 0xFF). |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9583 |
// Create mask to test for Unicode chars inside zmm vector |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9584 |
movl(result, 0x00FF); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9585 |
evpbroadcastw(tmp2Reg, result, Assembler::AVX_512bit); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9586 |
|
42587
80c66fce4b3b
8165287: fix compress intrinsics to produce correct results with avx512
vdeshpande
parents:
42039
diff
changeset
|
9587 |
testl(len, -64); |
51419
c11be049acb8
8209511: C2 asserts with UseSSE < 4 and AVX enabled: "Label was never bound to a location, but it was used as a jmp target'
thartmann
parents:
51350
diff
changeset
|
9588 |
jcc(Assembler::zero, post_alignment); |
42587
80c66fce4b3b
8165287: fix compress intrinsics to produce correct results with avx512
vdeshpande
parents:
42039
diff
changeset
|
9589 |
|
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9590 |
movl(tmp5, dst); |
42587
80c66fce4b3b
8165287: fix compress intrinsics to produce correct results with avx512
vdeshpande
parents:
42039
diff
changeset
|
9591 |
andl(tmp5, (32 - 1)); |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9592 |
negl(tmp5); |
42587
80c66fce4b3b
8165287: fix compress intrinsics to produce correct results with avx512
vdeshpande
parents:
42039
diff
changeset
|
9593 |
andl(tmp5, (32 - 1)); |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9594 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9595 |
// bail out when there is nothing to be done |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9596 |
testl(tmp5, 0xFFFFFFFF); |
51419
c11be049acb8
8209511: C2 asserts with UseSSE < 4 and AVX enabled: "Label was never bound to a location, but it was used as a jmp target'
thartmann
parents:
51350
diff
changeset
|
9597 |
jcc(Assembler::zero, post_alignment); |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9598 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9599 |
// ~(~0 << len), where len is the # of remaining elements to process |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9600 |
movl(result, 0xFFFFFFFF); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9601 |
shlxl(result, result, tmp5); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9602 |
notl(result); |
51976
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
9603 |
kmovdl(k3, result); |
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
9604 |
|
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
9605 |
evmovdquw(tmp1Reg, k3, Address(src, 0), Assembler::AVX_512bit); |
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
9606 |
evpcmpuw(k2, k3, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit); |
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
9607 |
ktestd(k2, k3); |
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
9608 |
jcc(Assembler::carryClear, return_zero); |
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
9609 |
|
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
9610 |
evpmovwb(Address(dst, 0), k3, tmp1Reg, Assembler::AVX_512bit); |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9611 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9612 |
addptr(src, tmp5); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9613 |
addptr(src, tmp5); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9614 |
addptr(dst, tmp5); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9615 |
subl(len, tmp5); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9616 |
|
51419
c11be049acb8
8209511: C2 asserts with UseSSE < 4 and AVX enabled: "Label was never bound to a location, but it was used as a jmp target'
thartmann
parents:
51350
diff
changeset
|
9617 |
bind(post_alignment); |
c11be049acb8
8209511: C2 asserts with UseSSE < 4 and AVX enabled: "Label was never bound to a location, but it was used as a jmp target'
thartmann
parents:
51350
diff
changeset
|
9618 |
// end of alignment |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9619 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9620 |
movl(tmp5, len); |
42587
80c66fce4b3b
8165287: fix compress intrinsics to produce correct results with avx512
vdeshpande
parents:
42039
diff
changeset
|
9621 |
andl(tmp5, (32 - 1)); // tail count (in chars) |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9622 |
andl(len, ~(32 - 1)); // vector count (in chars) |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9623 |
jcc(Assembler::zero, copy_loop_tail); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9624 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9625 |
lea(src, Address(src, len, Address::times_2)); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9626 |
lea(dst, Address(dst, len, Address::times_1)); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9627 |
negptr(len); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9628 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9629 |
bind(copy_32_loop); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9630 |
evmovdquw(tmp1Reg, Address(src, len, Address::times_2), Assembler::AVX_512bit); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9631 |
evpcmpuw(k2, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9632 |
kortestdl(k2, k2); |
51976
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
9633 |
jcc(Assembler::carryClear, return_zero); |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9634 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9635 |
// All elements in current processed chunk are valid candidates for |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9636 |
// compression. Write a truncated byte elements to the memory. |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9637 |
evpmovwb(Address(dst, len, Address::times_1), tmp1Reg, Assembler::AVX_512bit); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9638 |
addptr(len, 32); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9639 |
jcc(Assembler::notZero, copy_32_loop); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9640 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9641 |
bind(copy_loop_tail); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9642 |
// bail out when there is nothing to be done |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9643 |
testl(tmp5, 0xFFFFFFFF); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9644 |
jcc(Assembler::zero, return_length); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9645 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9646 |
movl(len, tmp5); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9647 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9648 |
// ~(~0 << len), where len is the # of remaining elements to process |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9649 |
movl(result, 0xFFFFFFFF); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9650 |
shlxl(result, result, len); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9651 |
notl(result); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9652 |
|
51976
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
9653 |
kmovdl(k3, result); |
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
9654 |
|
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
9655 |
evmovdquw(tmp1Reg, k3, Address(src, 0), Assembler::AVX_512bit); |
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
9656 |
evpcmpuw(k2, k3, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit); |
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
9657 |
ktestd(k2, k3); |
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
9658 |
jcc(Assembler::carryClear, return_zero); |
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
9659 |
|
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
9660 |
evpmovwb(Address(dst, 0), k3, tmp1Reg, Assembler::AVX_512bit); |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9661 |
jmp(return_length); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9662 |
|
51419
c11be049acb8
8209511: C2 asserts with UseSSE < 4 and AVX enabled: "Label was never bound to a location, but it was used as a jmp target'
thartmann
parents:
51350
diff
changeset
|
9663 |
bind(below_threshold); |
c11be049acb8
8209511: C2 asserts with UseSSE < 4 and AVX enabled: "Label was never bound to a location, but it was used as a jmp target'
thartmann
parents:
51350
diff
changeset
|
9664 |
} |
c11be049acb8
8209511: C2 asserts with UseSSE < 4 and AVX enabled: "Label was never bound to a location, but it was used as a jmp target'
thartmann
parents:
51350
diff
changeset
|
9665 |
|
33628 | 9666 |
if (UseSSE42Intrinsics) { |
9667 |
Label copy_32_loop, copy_16, copy_tail; |
|
9668 |
||
9669 |
movl(result, len); |
|
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9670 |
|
33628 | 9671 |
movl(tmp5, 0xff00ff00); // create mask to test for Unicode chars in vectors |
9672 |
||
9673 |
// vectored compression |
|
9674 |
andl(len, 0xfffffff0); // vector count (in chars) |
|
9675 |
andl(result, 0x0000000f); // tail count (in chars) |
|
9676 |
testl(len, len); |
|
51633
21154cb84d2a
8209594: guarantee(this->is8bit(imm8)) failed: Short forward jump exceeds 8-bit offset
kvn
parents:
51464
diff
changeset
|
9677 |
jcc(Assembler::zero, copy_16); |
33628 | 9678 |
|
9679 |
// compress 16 chars per iter |
|
9680 |
movdl(tmp1Reg, tmp5); |
|
9681 |
pshufd(tmp1Reg, tmp1Reg, 0); // store Unicode mask in tmp1Reg |
|
9682 |
pxor(tmp4Reg, tmp4Reg); |
|
9683 |
||
9684 |
lea(src, Address(src, len, Address::times_2)); |
|
9685 |
lea(dst, Address(dst, len, Address::times_1)); |
|
9686 |
negptr(len); |
|
9687 |
||
9688 |
bind(copy_32_loop); |
|
9689 |
movdqu(tmp2Reg, Address(src, len, Address::times_2)); // load 1st 8 characters |
|
9690 |
por(tmp4Reg, tmp2Reg); |
|
9691 |
movdqu(tmp3Reg, Address(src, len, Address::times_2, 16)); // load next 8 characters |
|
9692 |
por(tmp4Reg, tmp3Reg); |
|
9693 |
ptest(tmp4Reg, tmp1Reg); // check for Unicode chars in next vector |
|
9694 |
jcc(Assembler::notZero, return_zero); |
|
9695 |
packuswb(tmp2Reg, tmp3Reg); // only ASCII chars; compress each to 1 byte |
|
9696 |
movdqu(Address(dst, len, Address::times_1), tmp2Reg); |
|
9697 |
addptr(len, 16); |
|
9698 |
jcc(Assembler::notZero, copy_32_loop); |
|
9699 |
||
9700 |
// compress next vector of 8 chars (if any) |
|
9701 |
bind(copy_16); |
|
9702 |
movl(len, result); |
|
9703 |
andl(len, 0xfffffff8); // vector count (in chars) |
|
9704 |
andl(result, 0x00000007); // tail count (in chars) |
|
9705 |
testl(len, len); |
|
9706 |
jccb(Assembler::zero, copy_tail); |
|
9707 |
||
9708 |
movdl(tmp1Reg, tmp5); |
|
9709 |
pshufd(tmp1Reg, tmp1Reg, 0); // store Unicode mask in tmp1Reg |
|
9710 |
pxor(tmp3Reg, tmp3Reg); |
|
9711 |
||
9712 |
movdqu(tmp2Reg, Address(src, 0)); |
|
9713 |
ptest(tmp2Reg, tmp1Reg); // check for Unicode chars in vector |
|
9714 |
jccb(Assembler::notZero, return_zero); |
|
9715 |
packuswb(tmp2Reg, tmp3Reg); // only LATIN1 chars; compress each to 1 byte |
|
9716 |
movq(Address(dst, 0), tmp2Reg); |
|
9717 |
addptr(src, 16); |
|
9718 |
addptr(dst, 8); |
|
9719 |
||
9720 |
bind(copy_tail); |
|
9721 |
movl(len, result); |
|
9722 |
} |
|
9723 |
// compress 1 char per iter |
|
9724 |
testl(len, len); |
|
9725 |
jccb(Assembler::zero, return_length); |
|
9726 |
lea(src, Address(src, len, Address::times_2)); |
|
9727 |
lea(dst, Address(dst, len, Address::times_1)); |
|
9728 |
negptr(len); |
|
9729 |
||
9730 |
bind(copy_chars_loop); |
|
9731 |
load_unsigned_short(result, Address(src, len, Address::times_2)); |
|
9732 |
testl(result, 0xff00); // check if Unicode char |
|
9733 |
jccb(Assembler::notZero, return_zero); |
|
9734 |
movb(Address(dst, len, Address::times_1), result); // ASCII char; compress to 1 byte |
|
9735 |
increment(len); |
|
9736 |
jcc(Assembler::notZero, copy_chars_loop); |
|
9737 |
||
9738 |
// if compression succeeded, return length |
|
9739 |
bind(return_length); |
|
9740 |
pop(result); |
|
9741 |
jmpb(done); |
|
9742 |
||
9743 |
// if compression failed, return 0 |
|
9744 |
bind(return_zero); |
|
9745 |
xorl(result, result); |
|
9746 |
addptr(rsp, wordSize); |
|
9747 |
||
9748 |
bind(done); |
|
9749 |
} |
|
9750 |
||
9751 |
// Inflate byte[] array to char[]. |
|
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9752 |
// ..\jdk\src\java.base\share\classes\java\lang\StringLatin1.java |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9753 |
// @HotSpotIntrinsicCandidate |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9754 |
// private static void inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len) { |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9755 |
// for (int i = 0; i < len; i++) { |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9756 |
// dst[dstOff++] = (char)(src[srcOff++] & 0xff); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9757 |
// } |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9758 |
// } |
33628 | 9759 |
void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len, |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9760 |
XMMRegister tmp1, Register tmp2) { |
58462
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
9761 |
Label copy_chars_loop, done, below_threshold, avx3_threshold; |
33628 | 9762 |
// rsi: src |
9763 |
// rdi: dst |
|
9764 |
// rdx: len |
|
9765 |
// rcx: tmp2 |
|
9766 |
||
9767 |
// rsi holds start addr of source byte[] to be inflated |
|
9768 |
// rdi holds start addr of destination char[] |
|
9769 |
// rdx holds length |
|
9770 |
assert_different_registers(src, dst, len, tmp2); |
|
58462
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
9771 |
movl(tmp2, len); |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9772 |
if ((UseAVX > 2) && // AVX512 |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9773 |
VM_Version::supports_avx512vlbw() && |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9774 |
VM_Version::supports_bmi2()) { |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9775 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9776 |
Label copy_32_loop, copy_tail; |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9777 |
Register tmp3_aliased = len; |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9778 |
|
51419
c11be049acb8
8209511: C2 asserts with UseSSE < 4 and AVX enabled: "Label was never bound to a location, but it was used as a jmp target'
thartmann
parents:
51350
diff
changeset
|
9779 |
// if length of the string is less than 16, handle it in an old fashioned way |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9780 |
testl(len, -16); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9781 |
jcc(Assembler::zero, below_threshold); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9782 |
|
58462
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
9783 |
testl(len, -1 * AVX3Threshold); |
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
9784 |
jcc(Assembler::zero, avx3_threshold); |
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
9785 |
|
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9786 |
// In order to use only one arithmetic operation for the main loop we use |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9787 |
// this pre-calculation |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9788 |
andl(tmp2, (32 - 1)); // tail count (in chars), 32 element wide loop |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9789 |
andl(len, -32); // vector count |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9790 |
jccb(Assembler::zero, copy_tail); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9791 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9792 |
lea(src, Address(src, len, Address::times_1)); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9793 |
lea(dst, Address(dst, len, Address::times_2)); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9794 |
negptr(len); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9795 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9796 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9797 |
// inflate 32 chars per iter |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9798 |
bind(copy_32_loop); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9799 |
vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_512bit); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9800 |
evmovdquw(Address(dst, len, Address::times_2), tmp1, Assembler::AVX_512bit); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9801 |
addptr(len, 32); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9802 |
jcc(Assembler::notZero, copy_32_loop); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9803 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9804 |
bind(copy_tail); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9805 |
// bail out when there is nothing to be done |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9806 |
testl(tmp2, -1); // we don't destroy the contents of tmp2 here |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9807 |
jcc(Assembler::zero, done); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9808 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9809 |
// ~(~0 << length), where length is the # of remaining elements to process |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9810 |
movl(tmp3_aliased, -1); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9811 |
shlxl(tmp3_aliased, tmp3_aliased, tmp2); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9812 |
notl(tmp3_aliased); |
51976
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
9813 |
kmovdl(k2, tmp3_aliased); |
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
9814 |
evpmovzxbw(tmp1, k2, Address(src, 0), Assembler::AVX_512bit); |
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
9815 |
evmovdquw(Address(dst, 0), k2, tmp1, Assembler::AVX_512bit); |
390f529f4f22
8211251: Default mask register for avx512 instructions
kvn
parents:
51857
diff
changeset
|
9816 |
|
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9817 |
jmp(done); |
58462
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
9818 |
bind(avx3_threshold); |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9819 |
} |
33628 | 9820 |
if (UseSSE42Intrinsics) { |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9821 |
Label copy_16_loop, copy_8_loop, copy_bytes, copy_new_tail, copy_tail; |
33628 | 9822 |
|
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9823 |
if (UseAVX > 1) { |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9824 |
andl(tmp2, (16 - 1)); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9825 |
andl(len, -16); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9826 |
jccb(Assembler::zero, copy_new_tail); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9827 |
} else { |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9828 |
andl(tmp2, 0x00000007); // tail count (in chars) |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9829 |
andl(len, 0xfffffff8); // vector count (in chars) |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9830 |
jccb(Assembler::zero, copy_tail); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9831 |
} |
33628 | 9832 |
|
9833 |
// vectored inflation |
|
9834 |
lea(src, Address(src, len, Address::times_1)); |
|
9835 |
lea(dst, Address(dst, len, Address::times_2)); |
|
9836 |
negptr(len); |
|
9837 |
||
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9838 |
if (UseAVX > 1) { |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9839 |
bind(copy_16_loop); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9840 |
vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_256bit); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9841 |
vmovdqu(Address(dst, len, Address::times_2), tmp1); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9842 |
addptr(len, 16); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9843 |
jcc(Assembler::notZero, copy_16_loop); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9844 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9845 |
bind(below_threshold); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9846 |
bind(copy_new_tail); |
58462
c6f1226cfb72
8221092: UseAVX=3 has performance degredation on Skylake (X7) processors
vdeshpande
parents:
58421
diff
changeset
|
9847 |
movl(len, tmp2); |
38239
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9848 |
andl(tmp2, 0x00000007); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9849 |
andl(len, 0xFFFFFFF8); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9850 |
jccb(Assembler::zero, copy_tail); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9851 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9852 |
pmovzxbw(tmp1, Address(src, 0)); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9853 |
movdqu(Address(dst, 0), tmp1); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9854 |
addptr(src, 8); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9855 |
addptr(dst, 2 * 8); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9856 |
|
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9857 |
jmp(copy_tail, true); |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9858 |
} |
4d8b8ba74fea
8154974: AVX-512 equipped inflate, has_negatives & compress intrinsics
kvn
parents:
38138
diff
changeset
|
9859 |
|
33628 | 9860 |
// inflate 8 chars per iter |
9861 |
bind(copy_8_loop); |
|
9862 |
pmovzxbw(tmp1, Address(src, len, Address::times_1)); // unpack to 8 words |
|
9863 |
movdqu(Address(dst, len, Address::times_2), tmp1); |
|
9864 |
addptr(len, 8); |
|
9865 |
jcc(Assembler::notZero, copy_8_loop); |
|
9866 |
||
9867 |
bind(copy_tail); |
|
9868 |
movl(len, tmp2); |
|
9869 |
||
9870 |
cmpl(len, 4); |
|
9871 |
jccb(Assembler::less, copy_bytes); |
|
9872 |
||
9873 |
movdl(tmp1, Address(src, 0)); // load 4 byte chars |
|
9874 |
pmovzxbw(tmp1, tmp1); |
|
9875 |
movq(Address(dst, 0), tmp1); |
|
9876 |
subptr(len, 4); |
|
9877 |
addptr(src, 4); |
|
9878 |
addptr(dst, 8); |
|
9879 |
||
9880 |
bind(copy_bytes); |
|
51419
c11be049acb8
8209511: C2 asserts with UseSSE < 4 and AVX enabled: "Label was never bound to a location, but it was used as a jmp target'
thartmann
parents:
51350
diff
changeset
|
9881 |
} else { |
c11be049acb8
8209511: C2 asserts with UseSSE < 4 and AVX enabled: "Label was never bound to a location, but it was used as a jmp target'
thartmann
parents:
51350
diff
changeset
|
9882 |
bind(below_threshold); |
c11be049acb8
8209511: C2 asserts with UseSSE < 4 and AVX enabled: "Label was never bound to a location, but it was used as a jmp target'
thartmann
parents:
51350
diff
changeset
|
9883 |
} |
c11be049acb8
8209511: C2 asserts with UseSSE < 4 and AVX enabled: "Label was never bound to a location, but it was used as a jmp target'
thartmann
parents:
51350
diff
changeset
|
9884 |
|
33628 | 9885 |
testl(len, len); |
9886 |
jccb(Assembler::zero, done); |
|
9887 |
lea(src, Address(src, len, Address::times_1)); |
|
9888 |
lea(dst, Address(dst, len, Address::times_2)); |
|
9889 |
negptr(len); |
|
9890 |
||
9891 |
// inflate 1 char per iter |
|
9892 |
bind(copy_chars_loop); |
|
9893 |
load_unsigned_byte(tmp2, Address(src, len, Address::times_1)); // load byte char |
|
9894 |
movw(Address(dst, len, Address::times_2), tmp2); // inflate byte char to word |
|
9895 |
increment(len); |
|
9896 |
jcc(Assembler::notZero, copy_chars_loop); |
|
9897 |
||
9898 |
bind(done); |
|
9899 |
} |
|
9900 |
||
57804 | 9901 |
#ifdef _LP64 |
9902 |
void MacroAssembler::cache_wb(Address line) |
|
9903 |
{ |
|
9904 |
// 64 bit cpus always support clflush |
|
9905 |
assert(VM_Version::supports_clflush(), "clflush should be available"); |
|
9906 |
bool optimized = VM_Version::supports_clflushopt(); |
|
9907 |
bool no_evict = VM_Version::supports_clwb(); |
|
9908 |
||
9909 |
// prefer clwb (writeback without evict) otherwise |
|
9910 |
// prefer clflushopt (potentially parallel writeback with evict) |
|
9911 |
// otherwise fallback on clflush (serial writeback with evict) |
|
9912 |
||
9913 |
if (optimized) { |
|
9914 |
if (no_evict) { |
|
9915 |
clwb(line); |
|
9916 |
} else { |
|
9917 |
clflushopt(line); |
|
9918 |
} |
|
9919 |
} else { |
|
9920 |
// no need for fence when using CLFLUSH |
|
9921 |
clflush(line); |
|
9922 |
} |
|
9923 |
} |
|
9924 |
||
9925 |
void MacroAssembler::cache_wbsync(bool is_pre) |
|
9926 |
{ |
|
9927 |
assert(VM_Version::supports_clflush(), "clflush should be available"); |
|
9928 |
bool optimized = VM_Version::supports_clflushopt(); |
|
9929 |
bool no_evict = VM_Version::supports_clwb(); |
|
9930 |
||
9931 |
// pick the correct implementation |
|
9932 |
||
9933 |
if (!is_pre && (optimized || no_evict)) { |
|
9934 |
// need an sfence for post flush when using clflushopt or clwb |
|
9935 |
// otherwise no no need for any synchroniaztion |
|
9936 |
||
9937 |
sfence(); |
|
9938 |
} |
|
9939 |
} |
|
9940 |
#endif // _LP64 |
|
9941 |
||
14626 | 9942 |
Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) { |
9943 |
switch (cond) { |
|
9944 |
// Note some conditions are synonyms for others |
|
9945 |
case Assembler::zero: return Assembler::notZero; |
|
9946 |
case Assembler::notZero: return Assembler::zero; |
|
9947 |
case Assembler::less: return Assembler::greaterEqual; |
|
9948 |
case Assembler::lessEqual: return Assembler::greater; |
|
9949 |
case Assembler::greater: return Assembler::lessEqual; |
|
9950 |
case Assembler::greaterEqual: return Assembler::less; |
|
9951 |
case Assembler::below: return Assembler::aboveEqual; |
|
9952 |
case Assembler::belowEqual: return Assembler::above; |
|
9953 |
case Assembler::above: return Assembler::belowEqual; |
|
9954 |
case Assembler::aboveEqual: return Assembler::below; |
|
9955 |
case Assembler::overflow: return Assembler::noOverflow; |
|
9956 |
case Assembler::noOverflow: return Assembler::overflow; |
|
9957 |
case Assembler::negative: return Assembler::positive; |
|
9958 |
case Assembler::positive: return Assembler::negative; |
|
9959 |
case Assembler::parity: return Assembler::noParity; |
|
9960 |
case Assembler::noParity: return Assembler::parity; |
|
9961 |
} |
|
9962 |
ShouldNotReachHere(); return Assembler::overflow; |
|
9963 |
} |
|
9964 |
||
9965 |
SkipIfEqual::SkipIfEqual( |
|
9966 |
MacroAssembler* masm, const bool* flag_addr, bool value) { |
|
9967 |
_masm = masm; |
|
9968 |
_masm->cmp8(ExternalAddress((address)flag_addr), value); |
|
9969 |
_masm->jcc(Assembler::equal, _label); |
|
9970 |
} |
|
9971 |
||
9972 |
SkipIfEqual::~SkipIfEqual() { |
|
9973 |
_masm->bind(_label); |
|
9974 |
} |
|
34633
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
9975 |
|
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
9976 |
// 32-bit Windows has its own fast-path implementation |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
9977 |
// of get_thread |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
9978 |
#if !defined(WIN32) || defined(_LP64) |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
9979 |
|
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
9980 |
// This is simply a call to Thread::current() |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
9981 |
void MacroAssembler::get_thread(Register thread) { |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
9982 |
if (thread != rax) { |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
9983 |
push(rax); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
9984 |
} |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
9985 |
LP64_ONLY(push(rdi);) |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
9986 |
LP64_ONLY(push(rsi);) |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
9987 |
push(rdx); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
9988 |
push(rcx); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
9989 |
#ifdef _LP64 |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
9990 |
push(r8); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
9991 |
push(r9); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
9992 |
push(r10); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
9993 |
push(r11); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
9994 |
#endif |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
9995 |
|
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
9996 |
MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
9997 |
|
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
9998 |
#ifdef _LP64 |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
9999 |
pop(r11); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
10000 |
pop(r10); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
10001 |
pop(r9); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
10002 |
pop(r8); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
10003 |
#endif |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
10004 |
pop(rcx); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
10005 |
pop(rdx); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
10006 |
LP64_ONLY(pop(rsi);) |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
10007 |
LP64_ONLY(pop(rdi);) |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
10008 |
if (thread != rax) { |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
10009 |
mov(thread, rax); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
10010 |
pop(rax); |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
10011 |
} |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
10012 |
} |
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
10013 |
|
2a6c7c7b30a7
8132510: Replace ThreadLocalStorage with compiler/language-based thread-local variables
dholmes
parents:
34211
diff
changeset
|
10014 |
#endif |