50609
|
1 |
/*
|
|
2 |
* Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved.
|
|
3 |
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
|
|
4 |
*
|
|
5 |
* This code is free software; you can redistribute it and/or modify it
|
|
6 |
* under the terms of the GNU General Public License version 2 only, as
|
|
7 |
* published by the Free Software Foundation.
|
|
8 |
*
|
|
9 |
* This code is distributed in the hope that it will be useful, but WITHOUT
|
|
10 |
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
11 |
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
12 |
* version 2 for more details (a copy is included in the LICENSE file that
|
|
13 |
* accompanied this code).
|
|
14 |
*
|
|
15 |
* You should have received a copy of the GNU General Public License version
|
|
16 |
* 2 along with this work; if not, write to the Free Software Foundation,
|
|
17 |
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
18 |
*
|
|
19 |
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
|
|
20 |
* or visit www.oracle.com if you need additional information or have any
|
|
21 |
* questions.
|
|
22 |
*/
|
50858
|
23 |
|
|
24 |
|
50609
|
25 |
package org.graalvm.compiler.lir.amd64.vector;
|
|
26 |
|
|
27 |
import static jdk.vm.ci.code.ValueUtil.asRegister;
|
|
28 |
import static jdk.vm.ci.code.ValueUtil.isRegister;
|
|
29 |
import static jdk.vm.ci.code.ValueUtil.isStackSlot;
|
51436
|
30 |
import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp.VMOVD;
|
|
31 |
import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp.VMOVDQU;
|
|
32 |
import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp.VMOVQ;
|
|
33 |
import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp.VMOVSD;
|
|
34 |
import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp.VMOVSS;
|
|
35 |
import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp.VMOVUPD;
|
|
36 |
import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp.VMOVUPS;
|
|
37 |
import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexRVMOp.VXORPD;
|
50609
|
38 |
import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.COMPOSITE;
|
|
39 |
import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.HINT;
|
|
40 |
import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG;
|
|
41 |
import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.STACK;
|
|
42 |
import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.UNINITIALIZED;
|
|
43 |
|
|
44 |
import org.graalvm.compiler.asm.amd64.AMD64Address;
|
51436
|
45 |
import org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp;
|
50609
|
46 |
import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler;
|
|
47 |
import org.graalvm.compiler.asm.amd64.AVXKind;
|
|
48 |
import org.graalvm.compiler.asm.amd64.AVXKind.AVXSize;
|
|
49 |
import org.graalvm.compiler.debug.GraalError;
|
|
50 |
import org.graalvm.compiler.lir.LIRFrameState;
|
|
51 |
import org.graalvm.compiler.lir.LIRInstructionClass;
|
|
52 |
import org.graalvm.compiler.lir.Opcode;
|
|
53 |
import org.graalvm.compiler.lir.StandardOp.LoadConstantOp;
|
|
54 |
import org.graalvm.compiler.lir.StandardOp.ValueMoveOp;
|
|
55 |
import org.graalvm.compiler.lir.amd64.AMD64AddressValue;
|
51436
|
56 |
import org.graalvm.compiler.lir.amd64.AMD64LIRInstruction;
|
50609
|
57 |
import org.graalvm.compiler.lir.amd64.AMD64Move;
|
|
58 |
import org.graalvm.compiler.lir.amd64.AMD64RestoreRegistersOp;
|
|
59 |
import org.graalvm.compiler.lir.amd64.AMD64SaveRegistersOp;
|
|
60 |
import org.graalvm.compiler.lir.asm.CompilationResultBuilder;
|
|
61 |
|
|
62 |
import jdk.vm.ci.amd64.AMD64Kind;
|
|
63 |
import jdk.vm.ci.code.Register;
|
|
64 |
import jdk.vm.ci.code.RegisterValue;
|
|
65 |
import jdk.vm.ci.code.StackSlot;
|
|
66 |
import jdk.vm.ci.meta.AllocatableValue;
|
|
67 |
import jdk.vm.ci.meta.Constant;
|
|
68 |
import jdk.vm.ci.meta.JavaConstant;
|
|
69 |
import jdk.vm.ci.meta.Value;
|
|
70 |
|
|
71 |
public class AMD64VectorMove {
|
|
72 |
|
|
73 |
@Opcode("VMOVE")
|
51436
|
74 |
public static final class MoveToRegOp extends AMD64LIRInstruction implements ValueMoveOp {
|
50609
|
75 |
public static final LIRInstructionClass<MoveToRegOp> TYPE = LIRInstructionClass.create(MoveToRegOp.class);
|
|
76 |
|
|
77 |
@Def({REG, HINT}) protected AllocatableValue result;
|
|
78 |
@Use({REG, STACK}) protected AllocatableValue input;
|
|
79 |
|
|
80 |
public MoveToRegOp(AllocatableValue result, AllocatableValue input) {
|
|
81 |
super(TYPE);
|
|
82 |
this.result = result;
|
|
83 |
this.input = input;
|
|
84 |
}
|
|
85 |
|
|
86 |
@Override
|
51436
|
87 |
public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) {
|
|
88 |
move(crb, masm, result, input);
|
50609
|
89 |
}
|
|
90 |
|
|
91 |
@Override
|
|
92 |
public AllocatableValue getInput() {
|
|
93 |
return input;
|
|
94 |
}
|
|
95 |
|
|
96 |
@Override
|
|
97 |
public AllocatableValue getResult() {
|
|
98 |
return result;
|
|
99 |
}
|
|
100 |
}
|
|
101 |
|
|
102 |
@Opcode("VMOVE")
|
51436
|
103 |
public static final class MoveFromRegOp extends AMD64LIRInstruction implements ValueMoveOp {
|
50609
|
104 |
public static final LIRInstructionClass<MoveFromRegOp> TYPE = LIRInstructionClass.create(MoveFromRegOp.class);
|
|
105 |
|
|
106 |
@Def({REG, STACK}) protected AllocatableValue result;
|
|
107 |
@Use({REG, HINT}) protected AllocatableValue input;
|
|
108 |
|
|
109 |
public MoveFromRegOp(AllocatableValue result, AllocatableValue input) {
|
|
110 |
super(TYPE);
|
|
111 |
this.result = result;
|
|
112 |
this.input = input;
|
|
113 |
}
|
|
114 |
|
|
115 |
@Override
|
51436
|
116 |
public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) {
|
|
117 |
move(crb, masm, result, input);
|
50609
|
118 |
}
|
|
119 |
|
|
120 |
@Override
|
|
121 |
public AllocatableValue getInput() {
|
|
122 |
return input;
|
|
123 |
}
|
|
124 |
|
|
125 |
@Override
|
|
126 |
public AllocatableValue getResult() {
|
|
127 |
return result;
|
|
128 |
}
|
|
129 |
}
|
|
130 |
|
|
131 |
@Opcode("VMOVE")
|
51436
|
132 |
public static class MoveFromConstOp extends AMD64LIRInstruction implements LoadConstantOp {
|
50609
|
133 |
public static final LIRInstructionClass<MoveFromConstOp> TYPE = LIRInstructionClass.create(MoveFromConstOp.class);
|
|
134 |
|
|
135 |
@Def({REG, STACK}) protected AllocatableValue result;
|
|
136 |
private final JavaConstant input;
|
|
137 |
|
|
138 |
public MoveFromConstOp(AllocatableValue result, JavaConstant input) {
|
|
139 |
super(TYPE);
|
|
140 |
this.result = result;
|
|
141 |
this.input = input;
|
|
142 |
}
|
|
143 |
|
|
144 |
@Override
|
51436
|
145 |
public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) {
|
50609
|
146 |
if (isRegister(result)) {
|
51436
|
147 |
const2reg(crb, masm, (RegisterValue) result, input);
|
50609
|
148 |
} else {
|
|
149 |
assert isStackSlot(result);
|
51436
|
150 |
AMD64Move.const2stack(crb, masm, result, input);
|
50609
|
151 |
}
|
|
152 |
}
|
|
153 |
|
|
154 |
@Override
|
|
155 |
public Constant getConstant() {
|
|
156 |
return input;
|
|
157 |
}
|
|
158 |
|
|
159 |
@Override
|
|
160 |
public AllocatableValue getResult() {
|
|
161 |
return result;
|
|
162 |
}
|
|
163 |
}
|
|
164 |
|
|
165 |
@Opcode("VSTACKMOVE")
|
51436
|
166 |
public static final class StackMoveOp extends AMD64LIRInstruction implements ValueMoveOp {
|
50609
|
167 |
public static final LIRInstructionClass<StackMoveOp> TYPE = LIRInstructionClass.create(StackMoveOp.class);
|
|
168 |
|
|
169 |
@Def({STACK}) protected AllocatableValue result;
|
|
170 |
@Use({STACK, HINT}) protected AllocatableValue input;
|
|
171 |
@Alive({STACK, UNINITIALIZED}) private AllocatableValue backupSlot;
|
|
172 |
|
|
173 |
private Register scratch;
|
|
174 |
|
|
175 |
public StackMoveOp(AllocatableValue result, AllocatableValue input, Register scratch, AllocatableValue backupSlot) {
|
|
176 |
super(TYPE);
|
|
177 |
this.result = result;
|
|
178 |
this.input = input;
|
|
179 |
this.backupSlot = backupSlot;
|
|
180 |
this.scratch = scratch;
|
|
181 |
}
|
|
182 |
|
|
183 |
@Override
|
|
184 |
public AllocatableValue getInput() {
|
|
185 |
return input;
|
|
186 |
}
|
|
187 |
|
|
188 |
@Override
|
|
189 |
public AllocatableValue getResult() {
|
|
190 |
return result;
|
|
191 |
}
|
|
192 |
|
|
193 |
@Override
|
51436
|
194 |
public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) {
|
50609
|
195 |
// backup scratch register
|
|
196 |
move(crb, masm, backupSlot, scratch.asValue(backupSlot.getValueKind()));
|
|
197 |
// move stack slot
|
|
198 |
move(crb, masm, scratch.asValue(getInput().getValueKind()), getInput());
|
|
199 |
move(crb, masm, getResult(), scratch.asValue(getResult().getValueKind()));
|
|
200 |
// restore scratch register
|
|
201 |
move(crb, masm, scratch.asValue(backupSlot.getValueKind()), backupSlot);
|
|
202 |
|
|
203 |
}
|
|
204 |
}
|
|
205 |
|
51436
|
206 |
public abstract static class VectorMemOp extends AMD64LIRInstruction {
|
50609
|
207 |
|
|
208 |
protected final AVXSize size;
|
|
209 |
protected final VexMoveOp op;
|
|
210 |
|
|
211 |
@Use({COMPOSITE}) protected AMD64AddressValue address;
|
|
212 |
@State protected LIRFrameState state;
|
|
213 |
|
|
214 |
protected VectorMemOp(LIRInstructionClass<? extends VectorMemOp> c, AVXSize size, VexMoveOp op, AMD64AddressValue address, LIRFrameState state) {
|
|
215 |
super(c);
|
|
216 |
this.size = size;
|
|
217 |
this.op = op;
|
|
218 |
this.address = address;
|
|
219 |
this.state = state;
|
|
220 |
}
|
|
221 |
|
51436
|
222 |
protected abstract void emitMemAccess(AMD64MacroAssembler masm);
|
50609
|
223 |
|
|
224 |
@Override
|
51436
|
225 |
public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) {
|
50609
|
226 |
if (state != null) {
|
51436
|
227 |
crb.recordImplicitException(masm.position(), state);
|
50609
|
228 |
}
|
51436
|
229 |
emitMemAccess(masm);
|
50609
|
230 |
}
|
|
231 |
}
|
|
232 |
|
|
233 |
public static final class VectorLoadOp extends VectorMemOp {
|
|
234 |
public static final LIRInstructionClass<VectorLoadOp> TYPE = LIRInstructionClass.create(VectorLoadOp.class);
|
|
235 |
|
|
236 |
@Def({REG}) protected AllocatableValue result;
|
|
237 |
|
|
238 |
public VectorLoadOp(AVXSize size, VexMoveOp op, AllocatableValue result, AMD64AddressValue address, LIRFrameState state) {
|
|
239 |
super(TYPE, size, op, address, state);
|
|
240 |
this.result = result;
|
|
241 |
}
|
|
242 |
|
|
243 |
@Override
|
51436
|
244 |
public void emitMemAccess(AMD64MacroAssembler masm) {
|
|
245 |
op.emit(masm, size, asRegister(result), address.toAddress());
|
50609
|
246 |
}
|
|
247 |
}
|
|
248 |
|
|
249 |
public static class VectorStoreOp extends VectorMemOp {
|
|
250 |
public static final LIRInstructionClass<VectorStoreOp> TYPE = LIRInstructionClass.create(VectorStoreOp.class);
|
|
251 |
|
|
252 |
@Use({REG}) protected AllocatableValue input;
|
|
253 |
|
|
254 |
public VectorStoreOp(AVXSize size, VexMoveOp op, AMD64AddressValue address, AllocatableValue input, LIRFrameState state) {
|
|
255 |
super(TYPE, size, op, address, state);
|
|
256 |
this.input = input;
|
|
257 |
}
|
|
258 |
|
|
259 |
@Override
|
51436
|
260 |
public void emitMemAccess(AMD64MacroAssembler masm) {
|
|
261 |
op.emit(masm, size, address.toAddress(), asRegister(input));
|
50609
|
262 |
}
|
|
263 |
}
|
|
264 |
|
|
265 |
@Opcode("SAVE_REGISTER")
|
|
266 |
public static class SaveRegistersOp extends AMD64SaveRegistersOp {
|
|
267 |
public static final LIRInstructionClass<SaveRegistersOp> TYPE = LIRInstructionClass.create(SaveRegistersOp.class);
|
|
268 |
|
|
269 |
public SaveRegistersOp(Register[] savedRegisters, AllocatableValue[] slots, boolean supportsRemove) {
|
|
270 |
super(TYPE, savedRegisters, slots, supportsRemove);
|
|
271 |
}
|
|
272 |
|
|
273 |
@Override
|
|
274 |
protected void saveRegister(CompilationResultBuilder crb, AMD64MacroAssembler masm, StackSlot result, Register register) {
|
|
275 |
AMD64Kind kind = (AMD64Kind) result.getPlatformKind();
|
|
276 |
if (kind.isXMM()) {
|
|
277 |
VexMoveOp op;
|
|
278 |
if (kind.getVectorLength() > 1) {
|
|
279 |
op = getVectorMoveOp(kind.getScalar());
|
|
280 |
} else {
|
|
281 |
op = getScalarMoveOp(kind);
|
|
282 |
}
|
|
283 |
|
|
284 |
AMD64Address addr = (AMD64Address) crb.asAddress(result);
|
51436
|
285 |
op.emit(masm, AVXKind.getRegisterSize(kind), addr, register);
|
50609
|
286 |
} else {
|
|
287 |
super.saveRegister(crb, masm, result, register);
|
|
288 |
}
|
|
289 |
}
|
|
290 |
}
|
|
291 |
|
|
292 |
@Opcode("RESTORE_REGISTER")
|
|
293 |
public static final class RestoreRegistersOp extends AMD64RestoreRegistersOp {
|
|
294 |
public static final LIRInstructionClass<RestoreRegistersOp> TYPE = LIRInstructionClass.create(RestoreRegistersOp.class);
|
|
295 |
|
|
296 |
public RestoreRegistersOp(AllocatableValue[] source, AMD64SaveRegistersOp save) {
|
|
297 |
super(TYPE, source, save);
|
|
298 |
}
|
|
299 |
|
|
300 |
@Override
|
|
301 |
protected void restoreRegister(CompilationResultBuilder crb, AMD64MacroAssembler masm, Register register, StackSlot input) {
|
|
302 |
AMD64Kind kind = (AMD64Kind) input.getPlatformKind();
|
|
303 |
if (kind.isXMM()) {
|
|
304 |
VexMoveOp op;
|
|
305 |
if (kind.getVectorLength() > 1) {
|
|
306 |
op = getVectorMoveOp(kind.getScalar());
|
|
307 |
} else {
|
|
308 |
op = getScalarMoveOp(kind);
|
|
309 |
}
|
|
310 |
|
|
311 |
AMD64Address addr = (AMD64Address) crb.asAddress(input);
|
51436
|
312 |
op.emit(masm, AVXKind.getRegisterSize(kind), register, addr);
|
50609
|
313 |
} else {
|
|
314 |
super.restoreRegister(crb, masm, register, input);
|
|
315 |
}
|
|
316 |
}
|
|
317 |
}
|
|
318 |
|
|
319 |
private static VexMoveOp getScalarMoveOp(AMD64Kind kind) {
|
|
320 |
switch (kind) {
|
|
321 |
case SINGLE:
|
|
322 |
return VMOVSS;
|
|
323 |
case DOUBLE:
|
|
324 |
return VMOVSD;
|
|
325 |
default:
|
|
326 |
throw GraalError.shouldNotReachHere();
|
|
327 |
}
|
|
328 |
}
|
|
329 |
|
|
330 |
private static VexMoveOp getVectorMoveOp(AMD64Kind kind) {
|
|
331 |
switch (kind) {
|
|
332 |
case SINGLE:
|
|
333 |
return VMOVUPS;
|
|
334 |
case DOUBLE:
|
|
335 |
return VMOVUPD;
|
|
336 |
default:
|
|
337 |
return VMOVDQU;
|
|
338 |
}
|
|
339 |
}
|
|
340 |
|
|
341 |
private static VexMoveOp getVectorMemMoveOp(AMD64Kind kind) {
|
|
342 |
switch (AVXKind.getDataSize(kind)) {
|
|
343 |
case DWORD:
|
|
344 |
return VMOVD;
|
|
345 |
case QWORD:
|
|
346 |
return VMOVQ;
|
|
347 |
default:
|
|
348 |
return getVectorMoveOp(kind.getScalar());
|
|
349 |
}
|
|
350 |
}
|
|
351 |
|
51436
|
352 |
private static void move(CompilationResultBuilder crb, AMD64MacroAssembler masm, AllocatableValue result, Value input) {
|
50609
|
353 |
VexMoveOp op;
|
|
354 |
AVXSize size;
|
|
355 |
AMD64Kind kind = (AMD64Kind) result.getPlatformKind();
|
|
356 |
if (kind.getVectorLength() > 1) {
|
|
357 |
size = AVXKind.getRegisterSize(kind);
|
|
358 |
if (isRegister(input) && isRegister(result)) {
|
|
359 |
op = getVectorMoveOp(kind.getScalar());
|
|
360 |
} else {
|
|
361 |
op = getVectorMemMoveOp(kind);
|
|
362 |
}
|
|
363 |
} else {
|
|
364 |
size = AVXSize.XMM;
|
|
365 |
if (isRegister(input) && isRegister(result)) {
|
|
366 |
op = getVectorMoveOp(kind);
|
|
367 |
} else {
|
|
368 |
op = getScalarMoveOp(kind);
|
|
369 |
}
|
|
370 |
}
|
|
371 |
|
|
372 |
if (isRegister(input)) {
|
|
373 |
if (isRegister(result)) {
|
|
374 |
if (!asRegister(input).equals(asRegister(result))) {
|
51436
|
375 |
op.emit(masm, size, asRegister(result), asRegister(input));
|
50609
|
376 |
}
|
|
377 |
} else {
|
|
378 |
assert isStackSlot(result);
|
51436
|
379 |
op.emit(masm, size, (AMD64Address) crb.asAddress(result), asRegister(input));
|
50609
|
380 |
}
|
|
381 |
} else {
|
|
382 |
assert isStackSlot(input) && isRegister(result);
|
51436
|
383 |
op.emit(masm, size, asRegister(result), (AMD64Address) crb.asAddress(input));
|
50609
|
384 |
}
|
|
385 |
}
|
|
386 |
|
51436
|
387 |
private static void const2reg(CompilationResultBuilder crb, AMD64MacroAssembler masm, RegisterValue result, JavaConstant input) {
|
50609
|
388 |
if (input.isDefaultForKind()) {
|
|
389 |
AMD64Kind kind = (AMD64Kind) result.getPlatformKind();
|
|
390 |
Register register = result.getRegister();
|
51436
|
391 |
VXORPD.emit(masm, AVXKind.getRegisterSize(kind), register, register, register);
|
50609
|
392 |
return;
|
|
393 |
}
|
|
394 |
|
|
395 |
AMD64Address address;
|
|
396 |
switch (input.getJavaKind()) {
|
|
397 |
case Float:
|
|
398 |
address = (AMD64Address) crb.asFloatConstRef(input);
|
|
399 |
break;
|
|
400 |
|
|
401 |
case Double:
|
|
402 |
address = (AMD64Address) crb.asDoubleConstRef(input);
|
|
403 |
break;
|
|
404 |
|
|
405 |
default:
|
|
406 |
throw GraalError.shouldNotReachHere();
|
|
407 |
}
|
|
408 |
VexMoveOp op = getScalarMoveOp((AMD64Kind) result.getPlatformKind());
|
51436
|
409 |
op.emit(masm, AVXSize.XMM, asRegister(result), address);
|
50609
|
410 |
}
|
|
411 |
|
51436
|
412 |
public static final class AVXMoveToIntOp extends AMD64LIRInstruction {
|
50609
|
413 |
public static final LIRInstructionClass<AVXMoveToIntOp> TYPE = LIRInstructionClass.create(AVXMoveToIntOp.class);
|
|
414 |
|
|
415 |
@Opcode private final VexMoveOp opcode;
|
|
416 |
|
|
417 |
@Def({REG, STACK}) protected AllocatableValue result;
|
|
418 |
@Use({REG}) protected AllocatableValue input;
|
|
419 |
|
|
420 |
public AVXMoveToIntOp(VexMoveOp opcode, AllocatableValue result, AllocatableValue input) {
|
|
421 |
super(TYPE);
|
|
422 |
this.opcode = opcode;
|
|
423 |
this.result = result;
|
|
424 |
this.input = input;
|
|
425 |
}
|
|
426 |
|
|
427 |
@Override
|
51436
|
428 |
public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) {
|
50609
|
429 |
if (isRegister(result)) {
|
51436
|
430 |
opcode.emitReverse(masm, AVXSize.XMM, asRegister(result), asRegister(input));
|
50609
|
431 |
} else {
|
51436
|
432 |
opcode.emit(masm, AVXSize.XMM, (AMD64Address) crb.asAddress(result), asRegister(input));
|
50609
|
433 |
}
|
|
434 |
}
|
|
435 |
}
|
|
436 |
}
|