--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/hotspot/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64Move.java Thu Feb 16 15:46:09 2017 -0800
@@ -0,0 +1,561 @@
+/*
+ * Copyright (c) 2013, 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.compiler.lir.aarch64;
+
+import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.COMPOSITE;
+import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.HINT;
+import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG;
+import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.STACK;
+import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.UNINITIALIZED;
+import static org.graalvm.compiler.lir.LIRValueUtil.asJavaConstant;
+import static org.graalvm.compiler.lir.LIRValueUtil.isJavaConstant;
+import static jdk.vm.ci.aarch64.AArch64.sp;
+import static jdk.vm.ci.aarch64.AArch64.zr;
+import static jdk.vm.ci.code.ValueUtil.asAllocatableValue;
+import static jdk.vm.ci.code.ValueUtil.asRegister;
+import static jdk.vm.ci.code.ValueUtil.asStackSlot;
+import static jdk.vm.ci.code.ValueUtil.isRegister;
+import static jdk.vm.ci.code.ValueUtil.isStackSlot;
+
+import org.graalvm.compiler.asm.Label;
+import org.graalvm.compiler.asm.aarch64.AArch64Address;
+import org.graalvm.compiler.asm.aarch64.AArch64Assembler;
+import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler;
+import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler.ScratchRegister;
+import org.graalvm.compiler.core.common.LIRKind;
+import org.graalvm.compiler.core.common.type.DataPointerConstant;
+import org.graalvm.compiler.debug.GraalError;
+import org.graalvm.compiler.lir.LIRFrameState;
+import org.graalvm.compiler.lir.LIRInstructionClass;
+import org.graalvm.compiler.lir.Opcode;
+import org.graalvm.compiler.lir.StandardOp;
+import org.graalvm.compiler.lir.StandardOp.LoadConstantOp;
+import org.graalvm.compiler.lir.StandardOp.NullCheck;
+import org.graalvm.compiler.lir.StandardOp.ValueMoveOp;
+import org.graalvm.compiler.lir.VirtualStackSlot;
+import org.graalvm.compiler.lir.asm.CompilationResultBuilder;
+
+import jdk.vm.ci.aarch64.AArch64Kind;
+import jdk.vm.ci.code.Register;
+import jdk.vm.ci.code.StackSlot;
+import jdk.vm.ci.meta.AllocatableValue;
+import jdk.vm.ci.meta.Constant;
+import jdk.vm.ci.meta.JavaConstant;
+import jdk.vm.ci.meta.PlatformKind;
+import jdk.vm.ci.meta.Value;
+
+public class AArch64Move {
+
+ public static class LoadInlineConstant extends AArch64LIRInstruction implements LoadConstantOp {
+ public static final LIRInstructionClass<LoadInlineConstant> TYPE = LIRInstructionClass.create(LoadInlineConstant.class);
+
+ private JavaConstant constant;
+ @Def({REG, STACK}) AllocatableValue result;
+
+ public LoadInlineConstant(JavaConstant constant, AllocatableValue result) {
+ super(TYPE);
+ this.constant = constant;
+ this.result = result;
+ }
+
+ @Override
+ public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) {
+ if (isRegister(result)) {
+ const2reg(crb, masm, result, constant);
+ } else if (isStackSlot(result)) {
+ StackSlot slot = asStackSlot(result);
+ const2stack(crb, masm, slot, constant);
+ }
+ }
+
+ @Override
+ public Constant getConstant() {
+ return constant;
+ }
+
+ @Override
+ public AllocatableValue getResult() {
+ return result;
+ }
+ }
+
+ @Opcode("MOVE")
+ public static class Move extends AArch64LIRInstruction implements ValueMoveOp {
+ public static final LIRInstructionClass<Move> TYPE = LIRInstructionClass.create(Move.class);
+
+ @Def({REG, STACK, HINT}) protected AllocatableValue result;
+ @Use({REG, STACK}) protected AllocatableValue input;
+
+ public Move(AllocatableValue result, AllocatableValue input) {
+ super(TYPE);
+ this.result = result;
+ this.input = input;
+ }
+
+ @Override
+ public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) {
+ move(crb, masm, getResult(), getInput());
+ }
+
+ @Override
+ public AllocatableValue getInput() {
+ return input;
+ }
+
+ @Override
+ public AllocatableValue getResult() {
+ return result;
+ }
+ }
+
+ public static class LoadAddressOp extends AArch64LIRInstruction {
+ public static final LIRInstructionClass<LoadAddressOp> TYPE = LIRInstructionClass.create(LoadAddressOp.class);
+
+ @Def protected AllocatableValue result;
+ @Use(COMPOSITE) protected AArch64AddressValue address;
+
+ public LoadAddressOp(AllocatableValue result, AArch64AddressValue address) {
+ super(TYPE);
+ this.result = result;
+ this.address = address;
+ }
+
+ @Override
+ public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) {
+ Register dst = asRegister(result);
+ AArch64Address adr = address.toAddress();
+ masm.loadAddress(dst, adr, address.getPlatformKind().getSizeInBytes());
+ }
+ }
+
+ public static class LoadDataOp extends AArch64LIRInstruction {
+ public static final LIRInstructionClass<LoadDataOp> TYPE = LIRInstructionClass.create(LoadDataOp.class);
+
+ @Def protected AllocatableValue result;
+ private final DataPointerConstant data;
+
+ public LoadDataOp(AllocatableValue result, DataPointerConstant data) {
+ super(TYPE);
+ this.result = result;
+ this.data = data;
+ }
+
+ @Override
+ public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) {
+ Register dst = asRegister(result);
+ masm.loadAddress(dst, (AArch64Address) crb.recordDataReferenceInCode(data), data.getAlignment());
+ }
+ }
+
+ public static class StackLoadAddressOp extends AArch64LIRInstruction {
+ public static final LIRInstructionClass<StackLoadAddressOp> TYPE = LIRInstructionClass.create(StackLoadAddressOp.class);
+
+ @Def protected AllocatableValue result;
+ @Use({STACK, UNINITIALIZED}) protected AllocatableValue slot;
+
+ public StackLoadAddressOp(AllocatableValue result, AllocatableValue slot) {
+ super(TYPE);
+ assert slot instanceof VirtualStackSlot || slot instanceof StackSlot;
+ this.result = result;
+ this.slot = slot;
+ }
+
+ @Override
+ public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) {
+ AArch64Address address = (AArch64Address) crb.asAddress(slot);
+ PlatformKind kind = AArch64Kind.QWORD;
+ masm.loadAddress(asRegister(result, kind), address, kind.getSizeInBytes());
+ }
+ }
+
+ public static class MembarOp extends AArch64LIRInstruction {
+ public static final LIRInstructionClass<MembarOp> TYPE = LIRInstructionClass.create(MembarOp.class);
+
+ @SuppressWarnings("unused") private final int barriers;
+
+ public MembarOp(int barriers) {
+ super(TYPE);
+ this.barriers = barriers;
+ }
+
+ @Override
+ public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) {
+ // As I understand it load acquire/store release have the same semantics as on IA64
+ // and allow us to handle LoadStore, LoadLoad and StoreStore without an explicit
+ // barrier.
+ // But Graal support to figure out if a load/store is volatile is non-existant so for
+ // now
+ // just use
+ // memory barriers everywhere.
+ // if ((barrier & MemoryBarriers.STORE_LOAD) != 0) {
+ masm.dmb(AArch64MacroAssembler.BarrierKind.ANY_ANY);
+ // }
+ }
+ }
+
+ abstract static class MemOp extends AArch64LIRInstruction implements StandardOp.ImplicitNullCheck {
+
+ protected final AArch64Kind kind;
+ @Use({COMPOSITE}) protected AArch64AddressValue addressValue;
+ @State protected LIRFrameState state;
+
+ MemOp(LIRInstructionClass<? extends MemOp> c, AArch64Kind kind, AArch64AddressValue address, LIRFrameState state) {
+ super(c);
+ this.kind = kind;
+ this.addressValue = address;
+ this.state = state;
+ }
+
+ protected abstract void emitMemAccess(CompilationResultBuilder crb, AArch64MacroAssembler masm);
+
+ @Override
+ public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) {
+ if (state != null) {
+ crb.recordImplicitException(masm.position(), state);
+ }
+ emitMemAccess(crb, masm);
+ }
+
+ @Override
+ public boolean makeNullCheckFor(Value value, LIRFrameState nullCheckState, int implicitNullCheckLimit) {
+ int immediate = addressValue.getImmediate();
+ if (state == null && value.equals(addressValue.getBase()) && addressValue.getOffset().equals(Value.ILLEGAL) && immediate >= 0 && immediate < implicitNullCheckLimit) {
+ state = nullCheckState;
+ return true;
+ }
+ return false;
+ }
+ }
+
+ public static final class LoadOp extends MemOp {
+ public static final LIRInstructionClass<LoadOp> TYPE = LIRInstructionClass.create(LoadOp.class);
+
+ @Def protected AllocatableValue result;
+
+ public LoadOp(AArch64Kind kind, AllocatableValue result, AArch64AddressValue address, LIRFrameState state) {
+ super(TYPE, kind, address, state);
+ this.result = result;
+ }
+
+ @Override
+ protected void emitMemAccess(CompilationResultBuilder crb, AArch64MacroAssembler masm) {
+ AArch64Address address = addressValue.toAddress();
+ Register dst = asRegister(result);
+
+ int destSize = result.getPlatformKind().getSizeInBytes() * Byte.SIZE;
+ int srcSize = kind.getSizeInBytes() * Byte.SIZE;
+ if (kind.isInteger()) {
+ // TODO How to load unsigned chars without the necessary information?
+ masm.ldrs(destSize, srcSize, dst, address);
+ } else {
+ assert srcSize == destSize;
+ masm.fldr(srcSize, dst, address);
+ }
+ }
+ }
+
+ public static class StoreOp extends MemOp {
+ public static final LIRInstructionClass<StoreOp> TYPE = LIRInstructionClass.create(StoreOp.class);
+ @Use protected AllocatableValue input;
+
+ public StoreOp(AArch64Kind kind, AArch64AddressValue address, AllocatableValue input, LIRFrameState state) {
+ super(TYPE, kind, address, state);
+ this.input = input;
+ }
+
+ @Override
+ protected void emitMemAccess(CompilationResultBuilder crb, AArch64MacroAssembler masm) {
+ emitStore(crb, masm, kind, addressValue.toAddress(), input);
+ }
+ }
+
+ public static final class StoreConstantOp extends MemOp {
+ public static final LIRInstructionClass<StoreConstantOp> TYPE = LIRInstructionClass.create(StoreConstantOp.class);
+
+ protected final JavaConstant input;
+
+ public StoreConstantOp(AArch64Kind kind, AArch64AddressValue address, JavaConstant input, LIRFrameState state) {
+ super(TYPE, kind, address, state);
+ this.input = input;
+ if (!input.isDefaultForKind()) {
+ throw GraalError.shouldNotReachHere("Can only store null constants to memory");
+ }
+ }
+
+ @Override
+ public void emitMemAccess(CompilationResultBuilder crb, AArch64MacroAssembler masm) {
+ emitStore(crb, masm, kind, addressValue.toAddress(), zr.asValue(LIRKind.combine(addressValue)));
+ }
+ }
+
+ public static final class NullCheckOp extends AArch64LIRInstruction implements NullCheck {
+ public static final LIRInstructionClass<NullCheckOp> TYPE = LIRInstructionClass.create(NullCheckOp.class);
+
+ @Use(COMPOSITE) protected AArch64AddressValue address;
+ @State protected LIRFrameState state;
+
+ public NullCheckOp(AArch64AddressValue address, LIRFrameState state) {
+ super(TYPE);
+ this.address = address;
+ this.state = state;
+ }
+
+ @Override
+ public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) {
+ crb.recordImplicitException(masm.position(), state);
+ masm.ldr(64, zr, address.toAddress());
+ }
+
+ @Override
+ public Value getCheckedValue() {
+ return address.base;
+ }
+
+ @Override
+ public LIRFrameState getState() {
+ return state;
+ }
+ }
+
+ /**
+ * Compare and swap instruction. Does the following atomically: <code>
+ * CAS(newVal, expected, address):
+ * oldVal = *address
+ * if oldVal == expected:
+ * *address = newVal
+ * return oldVal
+ * </code>
+ */
+ @Opcode("CAS")
+ public static class CompareAndSwapOp extends AArch64LIRInstruction {
+ public static final LIRInstructionClass<CompareAndSwapOp> TYPE = LIRInstructionClass.create(CompareAndSwapOp.class);
+
+ @Def protected AllocatableValue resultValue;
+ @Alive protected Value expectedValue;
+ @Alive protected AllocatableValue newValue;
+ @Alive protected AllocatableValue addressValue;
+ @Temp protected AllocatableValue scratchValue;
+
+ public CompareAndSwapOp(AllocatableValue result, Value expectedValue, AllocatableValue newValue, AllocatableValue addressValue, AllocatableValue scratch) {
+ super(TYPE);
+ this.resultValue = result;
+ this.expectedValue = expectedValue;
+ this.newValue = newValue;
+ this.addressValue = addressValue;
+ this.scratchValue = scratch;
+ }
+
+ @Override
+ public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) {
+ AArch64Kind kind = (AArch64Kind) expectedValue.getPlatformKind();
+ assert kind.isInteger();
+ final int size = kind.getSizeInBytes() * Byte.SIZE;
+
+ Register address = asRegister(addressValue);
+ Register result = asRegister(resultValue);
+ Register newVal = asRegister(newValue);
+ Register scratch = asRegister(scratchValue);
+ // We could avoid using a scratch register here, by reusing resultValue for the stlxr
+ // success flag and issue a mov resultValue, expectedValue in case of success before
+ // returning.
+ Label retry = new Label();
+ Label fail = new Label();
+ masm.bind(retry);
+ masm.ldaxr(size, result, address);
+ AArch64Compare.gpCompare(masm, resultValue, expectedValue);
+ masm.branchConditionally(AArch64Assembler.ConditionFlag.NE, fail);
+ masm.stlxr(size, scratch, newVal, address);
+ // if scratch == 0 then write successful, else retry.
+ masm.cbnz(32, scratch, retry);
+ masm.bind(fail);
+ }
+ }
+
+ private static void emitStore(@SuppressWarnings("unused") CompilationResultBuilder crb, AArch64MacroAssembler masm, AArch64Kind kind, AArch64Address dst, Value src) {
+ int destSize = kind.getSizeInBytes() * Byte.SIZE;
+ if (kind.isInteger()) {
+ masm.str(destSize, asRegister(src), dst);
+ } else {
+ masm.fstr(destSize, asRegister(src), dst);
+ }
+ }
+
+ public static void move(CompilationResultBuilder crb, AArch64MacroAssembler masm, AllocatableValue result, Value input) {
+ if (isRegister(input)) {
+ if (isRegister(result)) {
+ reg2reg(crb, masm, result, asAllocatableValue(input));
+ } else if (isStackSlot(result)) {
+ reg2stack(crb, masm, result, asAllocatableValue(input));
+ } else {
+ throw GraalError.shouldNotReachHere();
+ }
+ } else if (isStackSlot(input)) {
+ if (isRegister(result)) {
+ stack2reg(crb, masm, result, asAllocatableValue(input));
+ } else {
+ throw GraalError.shouldNotReachHere();
+ }
+ } else if (isJavaConstant(input)) {
+ if (isRegister(result)) {
+ const2reg(crb, masm, result, asJavaConstant(input));
+ } else {
+ throw GraalError.shouldNotReachHere();
+ }
+ } else {
+ throw GraalError.shouldNotReachHere();
+ }
+ }
+
+ private static void reg2reg(@SuppressWarnings("unused") CompilationResultBuilder crb, AArch64MacroAssembler masm, AllocatableValue result, AllocatableValue input) {
+ Register dst = asRegister(result);
+ Register src = asRegister(input);
+ if (src.equals(dst)) {
+ return;
+ }
+ AArch64Kind kind = (AArch64Kind) input.getPlatformKind();
+ int size = kind.getSizeInBytes() * Byte.SIZE;
+ if (kind.isInteger()) {
+ masm.mov(size, dst, src);
+ } else {
+ masm.fmov(size, dst, src);
+ }
+ }
+
+ private static void reg2stack(CompilationResultBuilder crb, AArch64MacroAssembler masm, AllocatableValue result, AllocatableValue input) {
+ AArch64Address dest = loadStackSlotAddress(crb, masm, asStackSlot(result), Value.ILLEGAL);
+ Register src = asRegister(input);
+ AArch64Kind kind = (AArch64Kind) input.getPlatformKind();
+ int size = kind.getSizeInBytes() * Byte.SIZE;
+ if (kind.isInteger()) {
+ masm.str(size, src, dest);
+ } else {
+ masm.fstr(size, src, dest);
+ }
+ }
+
+ private static void stack2reg(CompilationResultBuilder crb, AArch64MacroAssembler masm, AllocatableValue result, AllocatableValue input) {
+ AArch64Kind kind = (AArch64Kind) input.getPlatformKind();
+ final int size = kind.getSizeInBytes() * Byte.SIZE;
+ if (kind.isInteger()) {
+ AArch64Address src = loadStackSlotAddress(crb, masm, asStackSlot(input), result);
+ masm.ldr(size, asRegister(result), src);
+ } else {
+ try (ScratchRegister sc = masm.getScratchRegister()) {
+ AllocatableValue scratchRegisterValue = sc.getRegister().asValue(LIRKind.combine(input));
+ AArch64Address src = loadStackSlotAddress(crb, masm, asStackSlot(input), scratchRegisterValue);
+ masm.fldr(size, asRegister(result), src);
+ }
+ }
+ }
+
+ private static void const2reg(CompilationResultBuilder crb, AArch64MacroAssembler masm, Value result, JavaConstant input) {
+ Register dst = asRegister(result);
+ switch (input.getJavaKind().getStackKind()) {
+ case Int:
+ final int value = input.asInt();
+ int maskedValue;
+ switch (input.getJavaKind()) {
+ case Boolean:
+ case Byte:
+ maskedValue = value & 0xFF;
+ break;
+ case Char:
+ case Short:
+ maskedValue = value & 0xFFFF;
+ break;
+ case Int:
+ maskedValue = value;
+ break;
+ default:
+ throw GraalError.shouldNotReachHere();
+ }
+ masm.mov(dst, maskedValue);
+ break;
+ case Long:
+ masm.mov(dst, input.asLong());
+ break;
+ case Float:
+ if (AArch64MacroAssembler.isFloatImmediate(input.asFloat())) {
+ masm.fmov(32, dst, input.asFloat());
+ } else {
+ masm.fldr(32, dst, (AArch64Address) crb.asFloatConstRef(input));
+ }
+ break;
+ case Double:
+ if (AArch64MacroAssembler.isDoubleImmediate(input.asDouble())) {
+ masm.fmov(64, dst, input.asDouble());
+ } else {
+ masm.fldr(64, dst, (AArch64Address) crb.asDoubleConstRef(input));
+ }
+ break;
+ case Object:
+ if (input.isNull()) {
+ masm.mov(dst, 0);
+ } else if (crb.target.inlineObjects) {
+ crb.recordInlineDataInCode(input);
+ masm.movNativeAddress(dst, 0xDEADDEADDEADDEADL);
+ } else {
+ masm.ldr(64, dst, (AArch64Address) crb.recordDataReferenceInCode(input, 8));
+ }
+ break;
+ default:
+ throw GraalError.shouldNotReachHere("kind=" + input.getJavaKind().getStackKind());
+ }
+ }
+
+ private static void const2stack(CompilationResultBuilder crb, AArch64MacroAssembler masm, Value result, JavaConstant constant) {
+ if (constant.isDefaultForKind() || constant.isNull()) {
+ AArch64Address resultAddress = (AArch64Address) crb.asAddress(result);
+ emitStore(crb, masm, (AArch64Kind) result.getPlatformKind(), resultAddress, zr.asValue(LIRKind.combine(result)));
+ } else {
+ try (ScratchRegister sc = masm.getScratchRegister()) {
+ Value scratchRegisterValue = sc.getRegister().asValue(LIRKind.combine(result));
+ const2reg(crb, masm, scratchRegisterValue, constant);
+ AArch64Address resultAddress = (AArch64Address) crb.asAddress(result);
+ emitStore(crb, masm, (AArch64Kind) result.getPlatformKind(), resultAddress, scratchRegisterValue);
+ }
+ }
+ }
+
+ /**
+ * Returns AArch64Address of given StackSlot. We cannot use CompilationResultBuilder.asAddress
+ * since this calls AArch64MacroAssembler.makeAddress with displacements that may be larger than
+ * 9-bit signed, which cannot be handled by that method.
+ *
+ * Instead we create an address ourselves. We use scaled unsigned addressing since we know the
+ * transfersize, which gives us a 15-bit address range (for longs/doubles) respectively a 14-bit
+ * range (for everything else).
+ *
+ * @param scratch Scratch register that can be used to load address. If Value.ILLEGAL this
+ * instruction fails if we try to access a StackSlot that is too large to be loaded
+ * directly.
+ * @return AArch64Address of given StackSlot. Uses scratch register if necessary to do so.
+ */
+ private static AArch64Address loadStackSlotAddress(CompilationResultBuilder crb, AArch64MacroAssembler masm, StackSlot slot, AllocatableValue scratch) {
+ int displacement = crb.frameMap.offsetForStackSlot(slot);
+ int transferSize = slot.getPlatformKind().getSizeInBytes();
+ Register scratchReg = Value.ILLEGAL.equals(scratch) ? zr : asRegister(scratch);
+ return masm.makeAddress(sp, displacement, scratchReg, transferSize, /* allowOverwrite */false);
+ }
+
+}