8176506: C2: loop unswitching and unsafe accesses cause crash
Reviewed-by: vlivanov, mcberg, kvn, simonis
/*
* Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef CPU_ARM_VM_ASSEMBLER_ARM_64_HPP
#define CPU_ARM_VM_ASSEMBLER_ARM_64_HPP
enum AsmShift12 {
lsl0, lsl12
};
enum AsmPrefetchOp {
pldl1keep = 0b00000,
pldl1strm,
pldl2keep,
pldl2strm,
pldl3keep,
pldl3strm,
plil1keep = 0b01000,
plil1strm,
plil2keep,
plil2strm,
plil3keep,
plil3strm,
pstl1keep = 0b10000,
pstl1strm,
pstl2keep,
pstl2strm,
pstl3keep,
pstl3strm,
};
// Shifted register operand for data processing instructions.
class AsmOperand VALUE_OBJ_CLASS_SPEC {
private:
Register _reg;
AsmShift _shift;
int _shift_imm;
public:
AsmOperand(Register reg) {
assert(reg != SP, "SP is not allowed in shifted register operand");
_reg = reg;
_shift = lsl;
_shift_imm = 0;
}
AsmOperand(Register reg, AsmShift shift, int shift_imm) {
assert(reg != SP, "SP is not allowed in shifted register operand");
assert(shift_imm >= 0, "shift amount should be non-negative");
_reg = reg;
_shift = shift;
_shift_imm = shift_imm;
}
Register reg() const {
return _reg;
}
AsmShift shift() const {
return _shift;
}
int shift_imm() const {
return _shift_imm;
}
};
class Assembler : public AbstractAssembler {
public:
static const int LogInstructionSize = 2;
static const int InstructionSize = 1 << LogInstructionSize;
Assembler(CodeBuffer* code) : AbstractAssembler(code) {}
static inline AsmCondition inverse(AsmCondition cond) {
assert ((cond != al) && (cond != nv), "AL and NV conditions cannot be inversed");
return (AsmCondition)((int)cond ^ 1);
}
// Returns value of nzcv flags conforming to the given condition.
static inline int flags_for_condition(AsmCondition cond) {
switch(cond) { // NZCV
case mi: case lt: return 0b1000;
case eq: case le: return 0b0100;
case hs: case hi: return 0b0010;
case vs: return 0b0001;
default: return 0b0000;
}
}
// Immediate, encoded into logical instructions.
class LogicalImmediate {
private:
bool _encoded;
bool _is32bit;
int _immN;
int _immr;
int _imms;
static inline bool has_equal_subpatterns(uintx imm, int size);
static inline int least_pattern_size(uintx imm);
static inline int population_count(uintx x);
static inline uintx set_least_zeroes(uintx x);
#ifdef ASSERT
uintx decode();
#endif
void construct(uintx imm, bool is32);
public:
LogicalImmediate(uintx imm, bool is32 = false) { construct(imm, is32); }
// Returns true if given immediate can be used in AArch64 logical instruction.
bool is_encoded() const { return _encoded; }
bool is32bit() const { return _is32bit; }
int immN() const { assert(_encoded, "should be"); return _immN; }
int immr() const { assert(_encoded, "should be"); return _immr; }
int imms() const { assert(_encoded, "should be"); return _imms; }
};
// Immediate, encoded into arithmetic add/sub instructions.
class ArithmeticImmediate {
private:
bool _encoded;
int _imm;
AsmShift12 _shift;
public:
ArithmeticImmediate(intx x) {
if (is_unsigned_imm_in_range(x, 12, 0)) {
_encoded = true;
_imm = x;
_shift = lsl0;
} else if (is_unsigned_imm_in_range(x, 12, 12)) {
_encoded = true;
_imm = x >> 12;
_shift = lsl12;
} else {
_encoded = false;
}
}
ArithmeticImmediate(intx x, AsmShift12 sh) {
if (is_unsigned_imm_in_range(x, 12, 0)) {
_encoded = true;
_imm = x;
_shift = sh;
} else {
_encoded = false;
}
}
// Returns true if this immediate can be used in AArch64 arithmetic (add/sub/cmp/cmn) instructions.
bool is_encoded() const { return _encoded; }
int imm() const { assert(_encoded, "should be"); return _imm; }
AsmShift12 shift() const { assert(_encoded, "should be"); return _shift; }
};
static inline bool is_imm_in_range(intx value, int bits, int align_bits) {
intx sign_bits = (value >> (bits + align_bits - 1));
return ((value & right_n_bits(align_bits)) == 0) && ((sign_bits == 0) || (sign_bits == -1));
}
static inline int encode_imm(intx value, int bits, int align_bits, int low_bit_in_encoding) {
assert (is_imm_in_range(value, bits, align_bits), "immediate value is out of range");
return ((value >> align_bits) & right_n_bits(bits)) << low_bit_in_encoding;
}
static inline bool is_unsigned_imm_in_range(intx value, int bits, int align_bits) {
return (value >= 0) && ((value & right_n_bits(align_bits)) == 0) && ((value >> (align_bits + bits)) == 0);
}
static inline int encode_unsigned_imm(intx value, int bits, int align_bits, int low_bit_in_encoding) {
assert (is_unsigned_imm_in_range(value, bits, align_bits), "immediate value is out of range");
return (value >> align_bits) << low_bit_in_encoding;
}
static inline bool is_offset_in_range(intx offset, int bits) {
assert (bits == 14 || bits == 19 || bits == 26, "wrong bits number");
return is_imm_in_range(offset, bits, 2);
}
static inline int encode_offset(intx offset, int bits, int low_bit_in_encoding) {
return encode_imm(offset, bits, 2, low_bit_in_encoding);
}
// Returns true if given value can be used as immediate in arithmetic (add/sub/cmp/cmn) instructions.
static inline bool is_arith_imm_in_range(intx value) {
return ArithmeticImmediate(value).is_encoded();
}
// Load/store instructions
#define F(mnemonic, opc) \
void mnemonic(Register rd, address literal_addr) { \
intx offset = literal_addr - pc(); \
assert (opc != 0b01 || offset == 0 || ((uintx)literal_addr & 7) == 0, "ldr target should be aligned"); \
assert (is_offset_in_range(offset, 19), "offset is out of range"); \
emit_int32(opc << 30 | 0b011 << 27 | encode_offset(offset, 19, 5) | rd->encoding_with_zr()); \
}
F(ldr_w, 0b00)
F(ldr, 0b01)
F(ldrsw, 0b10)
#undef F
#define F(mnemonic, opc) \
void mnemonic(FloatRegister rt, address literal_addr) { \
intx offset = literal_addr - pc(); \
assert (offset == 0 || ((uintx)literal_addr & right_n_bits(2 + opc)) == 0, "ldr target should be aligned"); \
assert (is_offset_in_range(offset, 19), "offset is out of range"); \
emit_int32(opc << 30 | 0b011100 << 24 | encode_offset(offset, 19, 5) | rt->encoding()); \
}
F(ldr_s, 0b00)
F(ldr_d, 0b01)
F(ldr_q, 0b10)
#undef F
#define F(mnemonic, size, o2, L, o1, o0) \
void mnemonic(Register rt, Register rn) { \
emit_int32(size << 30 | 0b001000 << 24 | o2 << 23 | L << 22 | o1 << 21 | 0b11111 << 16 | \
o0 << 15 | 0b11111 << 10 | rn->encoding_with_sp() << 5 | rt->encoding_with_zr()); \
}
F(ldxrb, 0b00, 0, 1, 0, 0)
F(ldaxrb, 0b00, 0, 1, 0, 1)
F(ldarb, 0b00, 1, 1, 0, 1)
F(ldxrh, 0b01, 0, 1, 0, 0)
F(ldaxrh, 0b01, 0, 1, 0, 1)
F(ldarh, 0b01, 1, 1, 0, 1)
F(ldxr_w, 0b10, 0, 1, 0, 0)
F(ldaxr_w, 0b10, 0, 1, 0, 1)
F(ldar_w, 0b10, 1, 1, 0, 1)
F(ldxr, 0b11, 0, 1, 0, 0)
F(ldaxr, 0b11, 0, 1, 0, 1)
F(ldar, 0b11, 1, 1, 0, 1)
F(stlrb, 0b00, 1, 0, 0, 1)
F(stlrh, 0b01, 1, 0, 0, 1)
F(stlr_w, 0b10, 1, 0, 0, 1)
F(stlr, 0b11, 1, 0, 0, 1)
#undef F
#define F(mnemonic, size, o2, L, o1, o0) \
void mnemonic(Register rs, Register rt, Register rn) { \
assert (rs != rt, "should be different"); \
assert (rs != rn, "should be different"); \
emit_int32(size << 30 | 0b001000 << 24 | o2 << 23 | L << 22 | o1 << 21 | rs->encoding_with_zr() << 16 | \
o0 << 15 | 0b11111 << 10 | rn->encoding_with_sp() << 5 | rt->encoding_with_zr()); \
}
F(stxrb, 0b00, 0, 0, 0, 0)
F(stlxrb, 0b00, 0, 0, 0, 1)
F(stxrh, 0b01, 0, 0, 0, 0)
F(stlxrh, 0b01, 0, 0, 0, 1)
F(stxr_w, 0b10, 0, 0, 0, 0)
F(stlxr_w, 0b10, 0, 0, 0, 1)
F(stxr, 0b11, 0, 0, 0, 0)
F(stlxr, 0b11, 0, 0, 0, 1)
#undef F
#define F(mnemonic, size, o2, L, o1, o0) \
void mnemonic(Register rt, Register rt2, Register rn) { \
assert (rt != rt2, "should be different"); \
emit_int32(size << 30 | 0b001000 << 24 | o2 << 23 | L << 22 | o1 << 21 | 0b11111 << 16 | \
o0 << 15 | rt2->encoding_with_zr() << 10 | rn->encoding_with_sp() << 5 | rt->encoding_with_zr()); \
}
F(ldxp_w, 0b10, 0, 1, 1, 0)
F(ldaxp_w, 0b10, 0, 1, 1, 1)
F(ldxp, 0b11, 0, 1, 1, 0)
F(ldaxp, 0b11, 0, 1, 1, 1)
#undef F
#define F(mnemonic, size, o2, L, o1, o0) \
void mnemonic(Register rs, Register rt, Register rt2, Register rn) { \
assert (rs != rt, "should be different"); \
assert (rs != rt2, "should be different"); \
assert (rs != rn, "should be different"); \
emit_int32(size << 30 | 0b001000 << 24 | o2 << 23 | L << 22 | o1 << 21 | rs->encoding_with_zr() << 16 | \
o0 << 15 | rt2->encoding_with_zr() << 10 | rn->encoding_with_sp() << 5 | rt->encoding_with_zr()); \
}
F(stxp_w, 0b10, 0, 0, 1, 0)
F(stlxp_w, 0b10, 0, 0, 1, 1)
F(stxp, 0b11, 0, 0, 1, 0)
F(stlxp, 0b11, 0, 0, 1, 1)
#undef F
#define F(mnemonic, opc, V, L) \
void mnemonic(Register rt, Register rt2, Register rn, int offset = 0) { \
assert (!L || rt != rt2, "should be different"); \
int align_bits = 2 + (opc >> 1); \
assert (is_imm_in_range(offset, 7, align_bits), "offset is out of range"); \
emit_int32(opc << 30 | 0b101 << 27 | V << 26 | L << 22 | encode_imm(offset, 7, align_bits, 15) | \
rt2->encoding_with_zr() << 10 | rn->encoding_with_sp() << 5 | rt->encoding_with_zr()); \
}
F(stnp_w, 0b00, 0, 0)
F(ldnp_w, 0b00, 0, 1)
F(stnp, 0b10, 0, 0)
F(ldnp, 0b10, 0, 1)
#undef F
#define F(mnemonic, opc, V, L) \
void mnemonic(FloatRegister rt, FloatRegister rt2, Register rn, int offset = 0) { \
assert (!L || (rt != rt2), "should be different"); \
int align_bits = 2 + opc; \
assert (is_imm_in_range(offset, 7, align_bits), "offset is out of range"); \
emit_int32(opc << 30 | 0b101 << 27 | V << 26 | L << 22 | encode_imm(offset, 7, align_bits, 15) | \
rt2->encoding() << 10 | rn->encoding_with_sp() << 5 | rt->encoding()); \
}
F(stnp_s, 0b00, 1, 0)
F(stnp_d, 0b01, 1, 0)
F(stnp_q, 0b10, 1, 0)
F(ldnp_s, 0b00, 1, 1)
F(ldnp_d, 0b01, 1, 1)
F(ldnp_q, 0b10, 1, 1)
#undef F
#define F(mnemonic, size, V, opc) \
void mnemonic(Register rt, Address addr) { \
assert((addr.mode() == basic_offset) || (rt != addr.base()), "should be different"); \
if (addr.index() == noreg) { \
if ((addr.mode() == basic_offset) && is_unsigned_imm_in_range(addr.disp(), 12, size)) { \
emit_int32(size << 30 | 0b111 << 27 | V << 26 | 0b01 << 24 | opc << 22 | \
encode_unsigned_imm(addr.disp(), 12, size, 10) | \
addr.base()->encoding_with_sp() << 5 | rt->encoding_with_zr()); \
} else { \
assert(is_imm_in_range(addr.disp(), 9, 0), "offset is out of range"); \
emit_int32(size << 30 | 0b111 << 27 | V << 26 | opc << 22 | encode_imm(addr.disp(), 9, 0, 12) | \
addr.mode() << 10 | addr.base()->encoding_with_sp() << 5 | rt->encoding_with_zr()); \
} \
} else { \
assert (addr.disp() == 0, "non-zero displacement for [reg + reg] address mode"); \
assert ((addr.shift_imm() == 0) || (addr.shift_imm() == size), "invalid shift amount"); \
emit_int32(size << 30 | 0b111 << 27 | V << 26 | opc << 22 | 1 << 21 | \
addr.index()->encoding_with_zr() << 16 | addr.extend() << 13 | (addr.shift_imm() != 0) << 12 | \
0b10 << 10 | addr.base()->encoding_with_sp() << 5 | rt->encoding_with_zr()); \
} \
}
F(strb, 0b00, 0, 0b00)
F(ldrb, 0b00, 0, 0b01)
F(ldrsb, 0b00, 0, 0b10)
F(ldrsb_w, 0b00, 0, 0b11)
F(strh, 0b01, 0, 0b00)
F(ldrh, 0b01, 0, 0b01)
F(ldrsh, 0b01, 0, 0b10)
F(ldrsh_w, 0b01, 0, 0b11)
F(str_w, 0b10, 0, 0b00)
F(ldr_w, 0b10, 0, 0b01)
F(ldrsw, 0b10, 0, 0b10)
F(str, 0b11, 0, 0b00)
F(ldr, 0b11, 0, 0b01)
#undef F
#define F(mnemonic, size, V, opc) \
void mnemonic(AsmPrefetchOp prfop, Address addr) { \
assert (addr.mode() == basic_offset, #mnemonic " supports only basic_offset address mode"); \
if (addr.index() == noreg) { \
if (is_unsigned_imm_in_range(addr.disp(), 12, size)) { \
emit_int32(size << 30 | 0b111 << 27 | V << 26 | 0b01 << 24 | opc << 22 | \
encode_unsigned_imm(addr.disp(), 12, size, 10) | \
addr.base()->encoding_with_sp() << 5 | prfop); \
} else { \
assert(is_imm_in_range(addr.disp(), 9, 0), "offset is out of range"); \
emit_int32(size << 30 | 0b111 << 27 | V << 26 | opc << 22 | encode_imm(addr.disp(), 9, 0, 12) | \
addr.base()->encoding_with_sp() << 5 | prfop); \
} \
} else { \
assert (addr.disp() == 0, "non-zero displacement for [reg + reg] address mode"); \
assert ((addr.shift_imm() == 0) || (addr.shift_imm() == size), "invalid shift amount"); \
emit_int32(size << 30 | 0b111 << 27 | V << 26 | opc << 22 | 1 << 21 | \
addr.index()->encoding_with_zr() << 16 | addr.extend() << 13 | (addr.shift_imm() != 0) << 12 | \
0b10 << 10 | addr.base()->encoding_with_sp() << 5 | prfop); \
} \
}
F(prfm, 0b11, 0, 0b10)
#undef F
#define F(mnemonic, size, V, opc) \
void mnemonic(FloatRegister rt, Address addr) { \
int align_bits = (((opc & 0b10) >> 1) << 2) | size; \
if (addr.index() == noreg) { \
if ((addr.mode() == basic_offset) && is_unsigned_imm_in_range(addr.disp(), 12, align_bits)) { \
emit_int32(size << 30 | 0b111 << 27 | V << 26 | 0b01 << 24 | opc << 22 | \
encode_unsigned_imm(addr.disp(), 12, align_bits, 10) | \
addr.base()->encoding_with_sp() << 5 | rt->encoding()); \
} else { \
assert(is_imm_in_range(addr.disp(), 9, 0), "offset is out of range"); \
emit_int32(size << 30 | 0b111 << 27 | V << 26 | opc << 22 | encode_imm(addr.disp(), 9, 0, 12) | \
addr.mode() << 10 | addr.base()->encoding_with_sp() << 5 | rt->encoding()); \
} \
} else { \
assert (addr.disp() == 0, "non-zero displacement for [reg + reg] address mode"); \
assert ((addr.shift_imm() == 0) || (addr.shift_imm() == align_bits), "invalid shift amount"); \
emit_int32(size << 30 | 0b111 << 27 | V << 26 | opc << 22 | 1 << 21 | \
addr.index()->encoding_with_zr() << 16 | addr.extend() << 13 | (addr.shift_imm() != 0) << 12 | \
0b10 << 10 | addr.base()->encoding_with_sp() << 5 | rt->encoding()); \
} \
}
F(str_b, 0b00, 1, 0b00)
F(ldr_b, 0b00, 1, 0b01)
F(str_h, 0b01, 1, 0b00)
F(ldr_h, 0b01, 1, 0b01)
F(str_s, 0b10, 1, 0b00)
F(ldr_s, 0b10, 1, 0b01)
F(str_d, 0b11, 1, 0b00)
F(ldr_d, 0b11, 1, 0b01)
F(str_q, 0b00, 1, 0b10)
F(ldr_q, 0b00, 1, 0b11)
#undef F
#define F(mnemonic, opc, V, L) \
void mnemonic(Register rt, Register rt2, Address addr) { \
assert((addr.mode() == basic_offset) || ((rt != addr.base()) && (rt2 != addr.base())), "should be different"); \
assert(!L || (rt != rt2), "should be different"); \
assert(addr.index() == noreg, "[reg + reg] address mode is not available for load/store pair"); \
int align_bits = 2 + (opc >> 1); \
int mode_encoding = (addr.mode() == basic_offset) ? 0b10 : addr.mode(); \
assert(is_imm_in_range(addr.disp(), 7, align_bits), "offset is out of range"); \
emit_int32(opc << 30 | 0b101 << 27 | V << 26 | mode_encoding << 23 | L << 22 | \
encode_imm(addr.disp(), 7, align_bits, 15) | rt2->encoding_with_zr() << 10 | \
addr.base()->encoding_with_sp() << 5 | rt->encoding_with_zr()); \
}
F(stp_w, 0b00, 0, 0)
F(ldp_w, 0b00, 0, 1)
F(ldpsw, 0b01, 0, 1)
F(stp, 0b10, 0, 0)
F(ldp, 0b10, 0, 1)
#undef F
#define F(mnemonic, opc, V, L) \
void mnemonic(FloatRegister rt, FloatRegister rt2, Address addr) { \
assert(!L || (rt != rt2), "should be different"); \
assert(addr.index() == noreg, "[reg + reg] address mode is not available for load/store pair"); \
int align_bits = 2 + opc; \
int mode_encoding = (addr.mode() == basic_offset) ? 0b10 : addr.mode(); \
assert(is_imm_in_range(addr.disp(), 7, align_bits), "offset is out of range"); \
emit_int32(opc << 30 | 0b101 << 27 | V << 26 | mode_encoding << 23 | L << 22 | \
encode_imm(addr.disp(), 7, align_bits, 15) | rt2->encoding() << 10 | \
addr.base()->encoding_with_sp() << 5 | rt->encoding()); \
}
F(stp_s, 0b00, 1, 0)
F(ldp_s, 0b00, 1, 1)
F(stp_d, 0b01, 1, 0)
F(ldp_d, 0b01, 1, 1)
F(stp_q, 0b10, 1, 0)
F(ldp_q, 0b10, 1, 1)
#undef F
// Data processing instructions
#define F(mnemonic, sf, opc) \
void mnemonic(Register rd, Register rn, const LogicalImmediate& imm) { \
assert (imm.is_encoded(), "illegal immediate for logical instruction"); \
assert (imm.is32bit() == (sf == 0), "immediate size does not match instruction size"); \
emit_int32(sf << 31 | opc << 29 | 0b100100 << 23 | imm.immN() << 22 | imm.immr() << 16 | \
imm.imms() << 10 | rn->encoding_with_zr() << 5 | \
((opc == 0b11) ? rd->encoding_with_zr() : rd->encoding_with_sp())); \
} \
void mnemonic(Register rd, Register rn, uintx imm) { \
LogicalImmediate limm(imm, (sf == 0)); \
mnemonic(rd, rn, limm); \
} \
void mnemonic(Register rd, Register rn, unsigned int imm) { \
mnemonic(rd, rn, (uintx)imm); \
}
F(andr_w, 0, 0b00)
F(orr_w, 0, 0b01)
F(eor_w, 0, 0b10)
F(ands_w, 0, 0b11)
F(andr, 1, 0b00)
F(orr, 1, 0b01)
F(eor, 1, 0b10)
F(ands, 1, 0b11)
#undef F
void tst(Register rn, unsigned int imm) {
ands(ZR, rn, imm);
}
void tst_w(Register rn, unsigned int imm) {
ands_w(ZR, rn, imm);
}
#define F(mnemonic, sf, opc, N) \
void mnemonic(Register rd, Register rn, AsmOperand operand) { \
assert (operand.shift_imm() >> (5 + sf) == 0, "shift amount is too large"); \
emit_int32(sf << 31 | opc << 29 | 0b01010 << 24 | operand.shift() << 22 | N << 21 | \
operand.reg()->encoding_with_zr() << 16 | operand.shift_imm() << 10 | \
rn->encoding_with_zr() << 5 | rd->encoding_with_zr()); \
}
F(andr_w, 0, 0b00, 0)
F(bic_w, 0, 0b00, 1)
F(orr_w, 0, 0b01, 0)
F(orn_w, 0, 0b01, 1)
F(eor_w, 0, 0b10, 0)
F(eon_w, 0, 0b10, 1)
F(ands_w, 0, 0b11, 0)
F(bics_w, 0, 0b11, 1)
F(andr, 1, 0b00, 0)
F(bic, 1, 0b00, 1)
F(orr, 1, 0b01, 0)
F(orn, 1, 0b01, 1)
F(eor, 1, 0b10, 0)
F(eon, 1, 0b10, 1)
F(ands, 1, 0b11, 0)
F(bics, 1, 0b11, 1)
#undef F
void tst(Register rn, AsmOperand operand) {
ands(ZR, rn, operand);
}
void tst_w(Register rn, AsmOperand operand) {
ands_w(ZR, rn, operand);
}
void mvn(Register rd, AsmOperand operand) {
orn(rd, ZR, operand);
}
void mvn_w(Register rd, AsmOperand operand) {
orn_w(rd, ZR, operand);
}
#define F(mnemonic, sf, op, S) \
void mnemonic(Register rd, Register rn, const ArithmeticImmediate& imm) { \
assert(imm.is_encoded(), "immediate is out of range"); \
emit_int32(sf << 31 | op << 30 | S << 29 | 0b10001 << 24 | imm.shift() << 22 | \
imm.imm() << 10 | rn->encoding_with_sp() << 5 | \
(S == 1 ? rd->encoding_with_zr() : rd->encoding_with_sp())); \
} \
void mnemonic(Register rd, Register rn, int imm) { \
mnemonic(rd, rn, ArithmeticImmediate(imm)); \
} \
void mnemonic(Register rd, Register rn, int imm, AsmShift12 shift) { \
mnemonic(rd, rn, ArithmeticImmediate(imm, shift)); \
} \
void mnemonic(Register rd, Register rn, Register rm, AsmExtendOp extend, int shift_imm = 0) { \
assert ((0 <= shift_imm) && (shift_imm <= 4), "shift amount is out of range"); \
emit_int32(sf << 31 | op << 30 | S << 29 | 0b01011001 << 21 | rm->encoding_with_zr() << 16 | \
extend << 13 | shift_imm << 10 | rn->encoding_with_sp() << 5 | \
(S == 1 ? rd->encoding_with_zr() : rd->encoding_with_sp())); \
} \
void mnemonic(Register rd, Register rn, AsmOperand operand) { \
assert (operand.shift() != ror, "illegal shift type"); \
assert (operand.shift_imm() >> (5 + sf) == 0, "shift amount is too large"); \
emit_int32(sf << 31 | op << 30 | S << 29 | 0b01011 << 24 | operand.shift() << 22 | \
operand.reg()->encoding_with_zr() << 16 | operand.shift_imm() << 10 | \
rn->encoding_with_zr() << 5 | rd->encoding_with_zr()); \
}
F(add_w, 0, 0, 0)
F(adds_w, 0, 0, 1)
F(sub_w, 0, 1, 0)
F(subs_w, 0, 1, 1)
F(add, 1, 0, 0)
F(adds, 1, 0, 1)
F(sub, 1, 1, 0)
F(subs, 1, 1, 1)
#undef F
void mov(Register rd, Register rm) {
if ((rd == SP) || (rm == SP)) {
add(rd, rm, 0);
} else {
orr(rd, ZR, rm);
}
}
void mov_w(Register rd, Register rm) {
if ((rd == SP) || (rm == SP)) {
add_w(rd, rm, 0);
} else {
orr_w(rd, ZR, rm);
}
}
void cmp(Register rn, int imm) {
subs(ZR, rn, imm);
}
void cmp_w(Register rn, int imm) {
subs_w(ZR, rn, imm);
}
void cmp(Register rn, Register rm) {
assert (rm != SP, "SP should not be used as the 2nd operand of cmp");
if (rn == SP) {
subs(ZR, rn, rm, ex_uxtx);
} else {
subs(ZR, rn, rm);
}
}
void cmp_w(Register rn, Register rm) {
assert ((rn != SP) && (rm != SP), "SP should not be used in 32-bit cmp");
subs_w(ZR, rn, rm);
}
void cmp(Register rn, AsmOperand operand) {
assert (rn != SP, "SP is not allowed in cmp with shifted register (AsmOperand)");
subs(ZR, rn, operand);
}
void cmn(Register rn, int imm) {
adds(ZR, rn, imm);
}
void cmn_w(Register rn, int imm) {
adds_w(ZR, rn, imm);
}
void cmn(Register rn, Register rm) {
assert (rm != SP, "SP should not be used as the 2nd operand of cmp");
if (rn == SP) {
adds(ZR, rn, rm, ex_uxtx);
} else {
adds(ZR, rn, rm);
}
}
void cmn_w(Register rn, Register rm) {
assert ((rn != SP) && (rm != SP), "SP should not be used in 32-bit cmp");
adds_w(ZR, rn, rm);
}
void neg(Register rd, Register rm) {
sub(rd, ZR, rm);
}
void neg_w(Register rd, Register rm) {
sub_w(rd, ZR, rm);
}
#define F(mnemonic, sf, op, S) \
void mnemonic(Register rd, Register rn, Register rm) { \
emit_int32(sf << 31 | op << 30 | S << 29 | 0b11010000 << 21 | rm->encoding_with_zr() << 16 | \
rn->encoding_with_zr() << 5 | rd->encoding_with_zr()); \
}
F(adc_w, 0, 0, 0)
F(adcs_w, 0, 0, 1)
F(sbc_w, 0, 1, 0)
F(sbcs_w, 0, 1, 1)
F(adc, 1, 0, 0)
F(adcs, 1, 0, 1)
F(sbc, 1, 1, 0)
F(sbcs, 1, 1, 1)
#undef F
#define F(mnemonic, sf, N) \
void mnemonic(Register rd, Register rn, Register rm, int lsb) { \
assert ((lsb >> (5 + sf)) == 0, "illegal least significant bit position"); \
emit_int32(sf << 31 | 0b100111 << 23 | N << 22 | rm->encoding_with_zr() << 16 | \
lsb << 10 | rn->encoding_with_zr() << 5 | rd->encoding_with_zr()); \
}
F(extr_w, 0, 0)
F(extr, 1, 1)
#undef F
#define F(mnemonic, sf, opc) \
void mnemonic(Register rd, int imm, int shift) { \
assert ((imm >> 16) == 0, "immediate is out of range"); \
assert (((shift & 0xf) == 0) && ((shift >> (5 + sf)) == 0), "invalid shift"); \
emit_int32(sf << 31 | opc << 29 | 0b100101 << 23 | (shift >> 4) << 21 | \
imm << 5 | rd->encoding_with_zr()); \
}
F(movn_w, 0, 0b00)
F(movz_w, 0, 0b10)
F(movk_w, 0, 0b11)
F(movn, 1, 0b00)
F(movz, 1, 0b10)
F(movk, 1, 0b11)
#undef F
void mov(Register rd, int imm) {
assert ((imm >> 16) == 0, "immediate is out of range");
movz(rd, imm, 0);
}
void mov_w(Register rd, int imm) {
assert ((imm >> 16) == 0, "immediate is out of range");
movz_w(rd, imm, 0);
}
#define F(mnemonic, sf, op, S) \
void mnemonic(Register rn, int imm, int nzcv, AsmCondition cond) { \
assert ((imm >> 5) == 0, "immediate is out of range"); \
assert ((nzcv >> 4) == 0, "illegal nzcv"); \
emit_int32(sf << 31 | op << 30 | S << 29 | 0b11010010 << 21 | imm << 16 | \
cond << 12 | 1 << 11 | rn->encoding_with_zr() << 5 | nzcv); \
}
F(ccmn_w, 0, 0, 1)
F(ccmp_w, 0, 1, 1)
F(ccmn, 1, 0, 1)
F(ccmp, 1, 1, 1)
#undef F
#define F(mnemonic, sf, op, S) \
void mnemonic(Register rn, Register rm, int nzcv, AsmCondition cond) { \
assert ((nzcv >> 4) == 0, "illegal nzcv"); \
emit_int32(sf << 31 | op << 30 | S << 29 | 0b11010010 << 21 | rm->encoding_with_zr() << 16 | \
cond << 12 | rn->encoding_with_zr() << 5 | nzcv); \
}
F(ccmn_w, 0, 0, 1)
F(ccmp_w, 0, 1, 1)
F(ccmn, 1, 0, 1)
F(ccmp, 1, 1, 1)
#undef F
#define F(mnemonic, sf, op, S, op2) \
void mnemonic(Register rd, Register rn, Register rm, AsmCondition cond) { \
emit_int32(sf << 31 | op << 30 | S << 29 | 0b11010100 << 21 | rm->encoding_with_zr() << 16 | \
cond << 12 | op2 << 10 | rn->encoding_with_zr() << 5 | rd->encoding_with_zr()); \
}
F(csel_w, 0, 0, 0, 0b00)
F(csinc_w, 0, 0, 0, 0b01)
F(csinv_w, 0, 1, 0, 0b00)
F(csneg_w, 0, 1, 0, 0b01)
F(csel, 1, 0, 0, 0b00)
F(csinc, 1, 0, 0, 0b01)
F(csinv, 1, 1, 0, 0b00)
F(csneg, 1, 1, 0, 0b01)
#undef F
void cset(Register rd, AsmCondition cond) {
csinc(rd, ZR, ZR, inverse(cond));
}
void cset_w(Register rd, AsmCondition cond) {
csinc_w(rd, ZR, ZR, inverse(cond));
}
void csetm(Register rd, AsmCondition cond) {
csinv(rd, ZR, ZR, inverse(cond));
}
void csetm_w(Register rd, AsmCondition cond) {
csinv_w(rd, ZR, ZR, inverse(cond));
}
void cinc(Register rd, Register rn, AsmCondition cond) {
csinc(rd, rn, rn, inverse(cond));
}
void cinc_w(Register rd, Register rn, AsmCondition cond) {
csinc_w(rd, rn, rn, inverse(cond));
}
void cinv(Register rd, Register rn, AsmCondition cond) {
csinv(rd, rn, rn, inverse(cond));
}
void cinv_w(Register rd, Register rn, AsmCondition cond) {
csinv_w(rd, rn, rn, inverse(cond));
}
#define F(mnemonic, sf, S, opcode) \
void mnemonic(Register rd, Register rn) { \
emit_int32(sf << 31 | 1 << 30 | S << 29 | 0b11010110 << 21 | opcode << 10 | \
rn->encoding_with_zr() << 5 | rd->encoding_with_zr()); \
}
F(rbit_w, 0, 0, 0b000000)
F(rev16_w, 0, 0, 0b000001)
F(rev_w, 0, 0, 0b000010)
F(clz_w, 0, 0, 0b000100)
F(cls_w, 0, 0, 0b000101)
F(rbit, 1, 0, 0b000000)
F(rev16, 1, 0, 0b000001)
F(rev32, 1, 0, 0b000010)
F(rev, 1, 0, 0b000011)
F(clz, 1, 0, 0b000100)
F(cls, 1, 0, 0b000101)
#undef F
#define F(mnemonic, sf, S, opcode) \
void mnemonic(Register rd, Register rn, Register rm) { \
emit_int32(sf << 31 | S << 29 | 0b11010110 << 21 | rm->encoding_with_zr() << 16 | \
opcode << 10 | rn->encoding_with_zr() << 5 | rd->encoding_with_zr()); \
}
F(udiv_w, 0, 0, 0b000010)
F(sdiv_w, 0, 0, 0b000011)
F(lslv_w, 0, 0, 0b001000)
F(lsrv_w, 0, 0, 0b001001)
F(asrv_w, 0, 0, 0b001010)
F(rorv_w, 0, 0, 0b001011)
F(udiv, 1, 0, 0b000010)
F(sdiv, 1, 0, 0b000011)
F(lslv, 1, 0, 0b001000)
F(lsrv, 1, 0, 0b001001)
F(asrv, 1, 0, 0b001010)
F(rorv, 1, 0, 0b001011)
#undef F
#define F(mnemonic, sf, op31, o0) \
void mnemonic(Register rd, Register rn, Register rm, Register ra) { \
emit_int32(sf << 31 | 0b11011 << 24 | op31 << 21 | rm->encoding_with_zr() << 16 | \
o0 << 15 | ra->encoding_with_zr() << 10 | rn->encoding_with_zr() << 5 | rd->encoding_with_zr()); \
}
F(madd_w, 0, 0b000, 0)
F(msub_w, 0, 0b000, 1)
F(madd, 1, 0b000, 0)
F(msub, 1, 0b000, 1)
F(smaddl, 1, 0b001, 0)
F(smsubl, 1, 0b001, 1)
F(umaddl, 1, 0b101, 0)
F(umsubl, 1, 0b101, 1)
#undef F
void mul(Register rd, Register rn, Register rm) {
madd(rd, rn, rm, ZR);
}
void mul_w(Register rd, Register rn, Register rm) {
madd_w(rd, rn, rm, ZR);
}
#define F(mnemonic, sf, op31, o0) \
void mnemonic(Register rd, Register rn, Register rm) { \
emit_int32(sf << 31 | 0b11011 << 24 | op31 << 21 | rm->encoding_with_zr() << 16 | \
o0 << 15 | 0b11111 << 10 | rn->encoding_with_zr() << 5 | rd->encoding_with_zr()); \
}
F(smulh, 1, 0b010, 0)
F(umulh, 1, 0b110, 0)
#undef F
#define F(mnemonic, op) \
void mnemonic(Register rd, address addr) { \
intx offset; \
if (op == 0) { \
offset = addr - pc(); \
} else { \
offset = (((intx)addr) - (((intx)pc()) & ~0xfff)) >> 12; \
} \
assert (is_imm_in_range(offset, 21, 0), "offset is out of range"); \
emit_int32(op << 31 | (offset & 3) << 29 | 0b10000 << 24 | \
encode_imm(offset >> 2, 19, 0, 5) | rd->encoding_with_zr()); \
} \
F(adr, 0)
F(adrp, 1)
#undef F
void adr(Register rd, Label& L) {
adr(rd, target(L));
}
#define F(mnemonic, sf, opc, N) \
void mnemonic(Register rd, Register rn, int immr, int imms) { \
assert ((immr >> (5 + sf)) == 0, "immr is out of range"); \
assert ((imms >> (5 + sf)) == 0, "imms is out of range"); \
emit_int32(sf << 31 | opc << 29 | 0b100110 << 23 | N << 22 | immr << 16 | \
imms << 10 | rn->encoding_with_zr() << 5 | rd->encoding_with_zr()); \
}
F(sbfm_w, 0, 0b00, 0)
F(bfm_w, 0, 0b01, 0)
F(ubfm_w, 0, 0b10, 0)
F(sbfm, 1, 0b00, 1)
F(bfm, 1, 0b01, 1)
F(ubfm, 1, 0b10, 1)
#undef F
#define F(alias, mnemonic, sf, immr, imms) \
void alias(Register rd, Register rn, int lsb, int width) { \
assert ((lsb >> (5 + sf)) == 0, "lsb is out of range"); \
assert ((1 <= width) && (width <= (32 << sf) - lsb), "width is out of range"); \
mnemonic(rd, rn, immr, imms); \
}
F(bfi_w, bfm_w, 0, (-lsb) & 0x1f, width - 1)
F(bfi, bfm, 1, (-lsb) & 0x3f, width - 1)
F(bfxil_w, bfm_w, 0, lsb, lsb + width - 1)
F(bfxil, bfm, 1, lsb, lsb + width - 1)
F(sbfiz_w, sbfm_w, 0, (-lsb) & 0x1f, width - 1)
F(sbfiz, sbfm, 1, (-lsb) & 0x3f, width - 1)
F(sbfx_w, sbfm_w, 0, lsb, lsb + width - 1)
F(sbfx, sbfm, 1, lsb, lsb + width - 1)
F(ubfiz_w, ubfm_w, 0, (-lsb) & 0x1f, width - 1)
F(ubfiz, ubfm, 1, (-lsb) & 0x3f, width - 1)
F(ubfx_w, ubfm_w, 0, lsb, lsb + width - 1)
F(ubfx, ubfm, 1, lsb, lsb + width - 1)
#undef F
#define F(alias, mnemonic, sf, immr, imms) \
void alias(Register rd, Register rn, int shift) { \
assert ((shift >> (5 + sf)) == 0, "shift is out of range"); \
mnemonic(rd, rn, immr, imms); \
}
F(_asr_w, sbfm_w, 0, shift, 31)
F(_asr, sbfm, 1, shift, 63)
F(_lsl_w, ubfm_w, 0, (-shift) & 0x1f, 31 - shift)
F(_lsl, ubfm, 1, (-shift) & 0x3f, 63 - shift)
F(_lsr_w, ubfm_w, 0, shift, 31)
F(_lsr, ubfm, 1, shift, 63)
#undef F
#define F(alias, mnemonic, immr, imms) \
void alias(Register rd, Register rn) { \
mnemonic(rd, rn, immr, imms); \
}
F(sxtb_w, sbfm_w, 0, 7)
F(sxtb, sbfm, 0, 7)
F(sxth_w, sbfm_w, 0, 15)
F(sxth, sbfm, 0, 15)
F(sxtw, sbfm, 0, 31)
F(uxtb_w, ubfm_w, 0, 7)
F(uxtb, ubfm, 0, 7)
F(uxth_w, ubfm_w, 0, 15)
F(uxth, ubfm, 0, 15)
#undef F
// Branch instructions
#define F(mnemonic, op) \
void mnemonic(Register rn) { \
emit_int32(0b1101011 << 25 | op << 21 | 0b11111 << 16 | rn->encoding_with_zr() << 5); \
}
F(br, 0b00)
F(blr, 0b01)
F(ret, 0b10)
#undef F
void ret() {
ret(LR);
}
#define F(mnemonic, op) \
void mnemonic(address target) { \
intx offset = target - pc(); \
assert (is_offset_in_range(offset, 26), "offset is out of range"); \
emit_int32(op << 31 | 0b00101 << 26 | encode_offset(offset, 26, 0)); \
}
F(b, 0)
F(bl, 1)
#undef F
void b(address target, AsmCondition cond) {
if (cond == al) {
b(target);
} else {
intx offset = target - pc();
assert (is_offset_in_range(offset, 19), "offset is out of range");
emit_int32(0b0101010 << 25 | encode_offset(offset, 19, 5) | cond);
}
}
#define F(mnemonic, sf, op) \
void mnemonic(Register rt, address target) { \
intx offset = target - pc(); \
assert (is_offset_in_range(offset, 19), "offset is out of range"); \
emit_int32(sf << 31 | 0b011010 << 25 | op << 24 | encode_offset(offset, 19, 5) | rt->encoding_with_zr()); \
} \
F(cbz_w, 0, 0)
F(cbnz_w, 0, 1)
F(cbz, 1, 0)
F(cbnz, 1, 1)
#undef F
#define F(mnemonic, op) \
void mnemonic(Register rt, int bit, address target) { \
intx offset = target - pc(); \
assert (is_offset_in_range(offset, 14), "offset is out of range"); \
assert (0 <= bit && bit < 64, "bit number is out of range"); \
emit_int32((bit >> 5) << 31 | 0b011011 << 25 | op << 24 | (bit & 0x1f) << 19 | \
encode_offset(offset, 14, 5) | rt->encoding_with_zr()); \
} \
F(tbz, 0)
F(tbnz, 1)
#undef F
// System instructions
enum DMB_Opt {
DMB_ld = 0b1101,
DMB_st = 0b1110,
DMB_all = 0b1111
};
#define F(mnemonic, L, op0, op1, CRn, op2, Rt) \
void mnemonic(DMB_Opt option) { \
emit_int32(0b1101010100 << 22 | L << 21 | op0 << 19 | op1 << 16 | \
CRn << 12 | option << 8 | op2 << 5 | Rt); \
}
F(dsb, 0, 0b00, 0b011, 0b0011, 0b100, 0b11111)
F(dmb, 0, 0b00, 0b011, 0b0011, 0b101, 0b11111)
#undef F
#define F(mnemonic, L, op0, op1, CRn, Rt) \
void mnemonic(int imm) { \
assert ((imm >> 7) == 0, "immediate is out of range"); \
emit_int32(0b1101010100 << 22 | L << 21 | op0 << 19 | op1 << 16 | \
CRn << 12 | imm << 5 | Rt); \
}
F(hint, 0, 0b00, 0b011, 0b0010, 0b11111)
#undef F
void nop() {
hint(0);
}
void yield() {
hint(1);
}
#define F(mnemonic, opc, op2, LL) \
void mnemonic(int imm = 0) { \
assert ((imm >> 16) == 0, "immediate is out of range"); \
emit_int32(0b11010100 << 24 | opc << 21 | imm << 5 | op2 << 2 | LL); \
}
F(brk, 0b001, 0b000, 0b00)
F(hlt, 0b010, 0b000, 0b00)
F(dpcs1, 0b101, 0b000, 0b01)
#undef F
enum SystemRegister { // o0<1> op1<3> CRn<4> CRm<4> op2<3>
SysReg_NZCV = 0b101101000010000,
SysReg_FPCR = 0b101101000100000,
};
void mrs(Register rt, SystemRegister systemReg) {
assert ((systemReg >> 15) == 0, "systemReg is out of range");
emit_int32(0b110101010011 << 20 | systemReg << 5 | rt->encoding_with_zr());
}
void msr(SystemRegister systemReg, Register rt) {
assert ((systemReg >> 15) == 0, "systemReg is out of range");
emit_int32(0b110101010001 << 20 | systemReg << 5 | rt->encoding_with_zr());
}
// Floating-point instructions
#define F(mnemonic, M, S, type, opcode2) \
void mnemonic(FloatRegister rn, FloatRegister rm) { \
emit_int32(M << 31 | S << 29 | 0b11110 << 24 | type << 22 | 1 << 21 | \
rm->encoding() << 16 | 0b1000 << 10 | rn->encoding() << 5 | opcode2); \
}
F(fcmp_s, 0, 0, 0b00, 0b00000)
F(fcmpe_s, 0, 0, 0b00, 0b01000)
F(fcmp_d, 0, 0, 0b01, 0b00000)
F(fcmpe_d, 0, 0, 0b01, 0b10000)
#undef F
#define F(mnemonic, M, S, type, opcode2) \
void mnemonic(FloatRegister rn) { \
emit_int32(M << 31 | S << 29 | 0b11110 << 24 | type << 22 | 1 << 21 | \
0b1000 << 10 | rn->encoding() << 5 | opcode2); \
}
F(fcmp0_s, 0, 0, 0b00, 0b01000)
F(fcmpe0_s, 0, 0, 0b00, 0b11000)
F(fcmp0_d, 0, 0, 0b01, 0b01000)
F(fcmpe0_d, 0, 0, 0b01, 0b11000)
#undef F
#define F(mnemonic, M, S, type, op) \
void mnemonic(FloatRegister rn, FloatRegister rm, int nzcv, AsmCondition cond) { \
assert ((nzcv >> 4) == 0, "illegal nzcv"); \
emit_int32(M << 31 | S << 29 | 0b11110 << 24 | type << 22 | 1 << 21 | \
rm->encoding() << 16 | cond << 12 | 0b01 << 10 | rn->encoding() << 5 | op << 4 | nzcv); \
}
F(fccmp_s, 0, 0, 0b00, 0)
F(fccmpe_s, 0, 0, 0b00, 1)
F(fccmp_d, 0, 0, 0b01, 0)
F(fccmpe_d, 0, 0, 0b01, 1)
#undef F
#define F(mnemonic, M, S, type) \
void mnemonic(FloatRegister rd, FloatRegister rn, FloatRegister rm, AsmCondition cond) { \
emit_int32(M << 31 | S << 29 | 0b11110 << 24 | type << 22 | 1 << 21 | \
rm->encoding() << 16 | cond << 12 | 0b11 << 10 | rn->encoding() << 5 | rd->encoding()); \
}
F(fcsel_s, 0, 0, 0b00)
F(fcsel_d, 0, 0, 0b01)
#undef F
#define F(mnemonic, M, S, type, opcode) \
void mnemonic(FloatRegister rd, FloatRegister rn) { \
emit_int32(M << 31 | S << 29 | 0b11110 << 24 | type << 22 | 1 << 21 | \
opcode << 15 | 0b10000 << 10 | rn->encoding() << 5 | rd->encoding()); \
}
F(fmov_s, 0, 0, 0b00, 0b000000)
F(fabs_s, 0, 0, 0b00, 0b000001)
F(fneg_s, 0, 0, 0b00, 0b000010)
F(fsqrt_s, 0, 0, 0b00, 0b000011)
F(fcvt_ds, 0, 0, 0b00, 0b000101)
F(fcvt_hs, 0, 0, 0b00, 0b000111)
F(frintn_s, 0, 0, 0b00, 0b001000)
F(frintp_s, 0, 0, 0b00, 0b001001)
F(frintm_s, 0, 0, 0b00, 0b001010)
F(frintz_s, 0, 0, 0b00, 0b001011)
F(frinta_s, 0, 0, 0b00, 0b001100)
F(frintx_s, 0, 0, 0b00, 0b001110)
F(frinti_s, 0, 0, 0b00, 0b001111)
F(fmov_d, 0, 0, 0b01, 0b000000)
F(fabs_d, 0, 0, 0b01, 0b000001)
F(fneg_d, 0, 0, 0b01, 0b000010)
F(fsqrt_d, 0, 0, 0b01, 0b000011)
F(fcvt_sd, 0, 0, 0b01, 0b000100)
F(fcvt_hd, 0, 0, 0b01, 0b000111)
F(frintn_d, 0, 0, 0b01, 0b001000)
F(frintp_d, 0, 0, 0b01, 0b001001)
F(frintm_d, 0, 0, 0b01, 0b001010)
F(frintz_d, 0, 0, 0b01, 0b001011)
F(frinta_d, 0, 0, 0b01, 0b001100)
F(frintx_d, 0, 0, 0b01, 0b001110)
F(frinti_d, 0, 0, 0b01, 0b001111)
F(fcvt_sh, 0, 0, 0b11, 0b000100)
F(fcvt_dh, 0, 0, 0b11, 0b000101)
#undef F
#define F(mnemonic, M, S, type, opcode) \
void mnemonic(FloatRegister rd, FloatRegister rn, FloatRegister rm) { \
emit_int32(M << 31 | S << 29 | 0b11110 << 24 | type << 22 | 1 << 21 | \
rm->encoding() << 16 | opcode << 12 | 0b10 << 10 | rn->encoding() << 5 | rd->encoding()); \
}
F(fmul_s, 0, 0, 0b00, 0b0000)
F(fdiv_s, 0, 0, 0b00, 0b0001)
F(fadd_s, 0, 0, 0b00, 0b0010)
F(fsub_s, 0, 0, 0b00, 0b0011)
F(fmax_s, 0, 0, 0b00, 0b0100)
F(fmin_s, 0, 0, 0b00, 0b0101)
F(fmaxnm_s, 0, 0, 0b00, 0b0110)
F(fminnm_s, 0, 0, 0b00, 0b0111)
F(fnmul_s, 0, 0, 0b00, 0b1000)
F(fmul_d, 0, 0, 0b01, 0b0000)
F(fdiv_d, 0, 0, 0b01, 0b0001)
F(fadd_d, 0, 0, 0b01, 0b0010)
F(fsub_d, 0, 0, 0b01, 0b0011)
F(fmax_d, 0, 0, 0b01, 0b0100)
F(fmin_d, 0, 0, 0b01, 0b0101)
F(fmaxnm_d, 0, 0, 0b01, 0b0110)
F(fminnm_d, 0, 0, 0b01, 0b0111)
F(fnmul_d, 0, 0, 0b01, 0b1000)
#undef F
#define F(mnemonic, M, S, type, o1, o0) \
void mnemonic(FloatRegister rd, FloatRegister rn, FloatRegister rm, FloatRegister ra) { \
emit_int32(M << 31 | S << 29 | 0b11111 << 24 | type << 22 | o1 << 21 | rm->encoding() << 16 | \
o0 << 15 | ra->encoding() << 10 | rn->encoding() << 5 | rd->encoding()); \
}
F(fmadd_s, 0, 0, 0b00, 0, 0)
F(fmsub_s, 0, 0, 0b00, 0, 1)
F(fnmadd_s, 0, 0, 0b00, 1, 0)
F(fnmsub_s, 0, 0, 0b00, 1, 1)
F(fmadd_d, 0, 0, 0b01, 0, 0)
F(fmsub_d, 0, 0, 0b01, 0, 1)
F(fnmadd_d, 0, 0, 0b01, 1, 0)
F(fnmsub_d, 0, 0, 0b01, 1, 1)
#undef F
#define F(mnemonic, M, S, type) \
void mnemonic(FloatRegister rd, int imm8) { \
assert ((imm8 >> 8) == 0, "immediate is out of range"); \
emit_int32(M << 31 | S << 29 | 0b11110 << 24 | type << 22 | 1 << 21 | \
imm8 << 13 | 0b100 << 10 | rd->encoding()); \
}
F(fmov_s, 0, 0, 0b00)
F(fmov_d, 0, 0, 0b01)
#undef F
#define F(mnemonic, sf, S, type, rmode, opcode) \
void mnemonic(Register rd, FloatRegister rn) { \
emit_int32(sf << 31 | S << 29 | 0b11110 << 24 | type << 22 | 1 << 21 | \
rmode << 19 | opcode << 16 | rn->encoding() << 5 | rd->encoding_with_zr()); \
}
F(fcvtns_ws, 0, 0, 0b00, 0b00, 0b000)
F(fcvtnu_ws, 0, 0, 0b00, 0b00, 0b001)
F(fcvtas_ws, 0, 0, 0b00, 0b00, 0b100)
F(fcvtau_ws, 0, 0, 0b00, 0b00, 0b101)
F(fmov_ws, 0, 0, 0b00, 0b00, 0b110)
F(fcvtps_ws, 0, 0, 0b00, 0b01, 0b000)
F(fcvtpu_ws, 0, 0, 0b00, 0b01, 0b001)
F(fcvtms_ws, 0, 0, 0b00, 0b10, 0b000)
F(fcvtmu_ws, 0, 0, 0b00, 0b10, 0b001)
F(fcvtzs_ws, 0, 0, 0b00, 0b11, 0b000)
F(fcvtzu_ws, 0, 0, 0b00, 0b11, 0b001)
F(fcvtns_wd, 0, 0, 0b01, 0b00, 0b000)
F(fcvtnu_wd, 0, 0, 0b01, 0b00, 0b001)
F(fcvtas_wd, 0, 0, 0b01, 0b00, 0b100)
F(fcvtau_wd, 0, 0, 0b01, 0b00, 0b101)
F(fcvtps_wd, 0, 0, 0b01, 0b01, 0b000)
F(fcvtpu_wd, 0, 0, 0b01, 0b01, 0b001)
F(fcvtms_wd, 0, 0, 0b01, 0b10, 0b000)
F(fcvtmu_wd, 0, 0, 0b01, 0b10, 0b001)
F(fcvtzs_wd, 0, 0, 0b01, 0b11, 0b000)
F(fcvtzu_wd, 0, 0, 0b01, 0b11, 0b001)
F(fcvtns_xs, 1, 0, 0b00, 0b00, 0b000)
F(fcvtnu_xs, 1, 0, 0b00, 0b00, 0b001)
F(fcvtas_xs, 1, 0, 0b00, 0b00, 0b100)
F(fcvtau_xs, 1, 0, 0b00, 0b00, 0b101)
F(fcvtps_xs, 1, 0, 0b00, 0b01, 0b000)
F(fcvtpu_xs, 1, 0, 0b00, 0b01, 0b001)
F(fcvtms_xs, 1, 0, 0b00, 0b10, 0b000)
F(fcvtmu_xs, 1, 0, 0b00, 0b10, 0b001)
F(fcvtzs_xs, 1, 0, 0b00, 0b11, 0b000)
F(fcvtzu_xs, 1, 0, 0b00, 0b11, 0b001)
F(fcvtns_xd, 1, 0, 0b01, 0b00, 0b000)
F(fcvtnu_xd, 1, 0, 0b01, 0b00, 0b001)
F(fcvtas_xd, 1, 0, 0b01, 0b00, 0b100)
F(fcvtau_xd, 1, 0, 0b01, 0b00, 0b101)
F(fmov_xd, 1, 0, 0b01, 0b00, 0b110)
F(fcvtps_xd, 1, 0, 0b01, 0b01, 0b000)
F(fcvtpu_xd, 1, 0, 0b01, 0b01, 0b001)
F(fcvtms_xd, 1, 0, 0b01, 0b10, 0b000)
F(fcvtmu_xd, 1, 0, 0b01, 0b10, 0b001)
F(fcvtzs_xd, 1, 0, 0b01, 0b11, 0b000)
F(fcvtzu_xd, 1, 0, 0b01, 0b11, 0b001)
F(fmov_xq, 1, 0, 0b10, 0b01, 0b110)
#undef F
#define F(mnemonic, sf, S, type, rmode, opcode) \
void mnemonic(FloatRegister rd, Register rn) { \
emit_int32(sf << 31 | S << 29 | 0b11110 << 24 | type << 22 | 1 << 21 | \
rmode << 19 | opcode << 16 | rn->encoding_with_zr() << 5 | rd->encoding()); \
}
F(scvtf_sw, 0, 0, 0b00, 0b00, 0b010)
F(ucvtf_sw, 0, 0, 0b00, 0b00, 0b011)
F(fmov_sw, 0, 0, 0b00, 0b00, 0b111)
F(scvtf_dw, 0, 0, 0b01, 0b00, 0b010)
F(ucvtf_dw, 0, 0, 0b01, 0b00, 0b011)
F(scvtf_sx, 1, 0, 0b00, 0b00, 0b010)
F(ucvtf_sx, 1, 0, 0b00, 0b00, 0b011)
F(scvtf_dx, 1, 0, 0b01, 0b00, 0b010)
F(ucvtf_dx, 1, 0, 0b01, 0b00, 0b011)
F(fmov_dx, 1, 0, 0b01, 0b00, 0b111)
F(fmov_qx, 1, 0, 0b10, 0b01, 0b111)
#undef F
#define F(mnemonic, opcode) \
void mnemonic(FloatRegister Vd, FloatRegister Vn) { \
emit_int32( opcode << 10 | Vn->encoding() << 5 | Vd->encoding()); \
}
F(aese, 0b0100111000101000010010);
F(aesd, 0b0100111000101000010110);
F(aesmc, 0b0100111000101000011010);
F(aesimc, 0b0100111000101000011110);
#undef F
#ifdef COMPILER2
typedef VFP::double_num double_num;
typedef VFP::float_num float_num;
#endif
void vcnt(FloatRegister Dd, FloatRegister Dn, int quad = 0, int size = 0) {
// emitted at VM startup to detect whether the instruction is available
assert(!VM_Version::is_initialized() || VM_Version::has_simd(), "simd instruction");
assert(size == 0, "illegal size value");
emit_int32(0x0e205800 | quad << 30 | size << 22 | Dn->encoding() << 5 | Dd->encoding());
}
#ifdef COMPILER2
void addv(FloatRegister Dd, FloatRegister Dm, int quad, int size) {
// emitted at VM startup to detect whether the instruction is available
assert(VM_Version::has_simd(), "simd instruction");
assert((quad & ~1) == 0, "illegal value");
assert(size >= 0 && size < 3, "illegal value");
assert(((size << 1) | quad) != 4, "illegal values (size 2, quad 0)");
emit_int32(0x0e31b800 | quad << 30 | size << 22 | Dm->encoding() << 5 | Dd->encoding());
}
enum VElem_Size {
VELEM_SIZE_8 = 0x00,
VELEM_SIZE_16 = 0x01,
VELEM_SIZE_32 = 0x02,
VELEM_SIZE_64 = 0x03
};
enum VLD_Type {
VLD1_TYPE_1_REG = 0b0111,
VLD1_TYPE_2_REGS = 0b1010,
VLD1_TYPE_3_REGS = 0b0110,
VLD1_TYPE_4_REGS = 0b0010
};
enum VFloat_Arith_Size {
VFA_SIZE_F32 = 0b0,
VFA_SIZE_F64 = 0b1
};
#define F(mnemonic, U, S, P) \
void mnemonic(FloatRegister fd, FloatRegister fn, FloatRegister fm, \
int size, int quad) { \
assert(VM_Version::has_simd(), "simd instruction"); \
assert(!(size == VFA_SIZE_F64 && !quad), "reserved"); \
assert((size & 1) == size, "overflow"); \
emit_int32(quad << 30 | U << 29 | 0b01110 << 24 | \
S << 23 | size << 22 | 1 << 21 | P << 11 | 1 << 10 | \
fm->encoding() << 16 | \
fn->encoding() << 5 | \
fd->encoding()); \
}
F(vaddF, 0, 0, 0b11010) // Vd = Vn + Vm (float)
F(vsubF, 0, 1, 0b11010) // Vd = Vn - Vm (float)
F(vmulF, 1, 0, 0b11011) // Vd = Vn - Vm (float)
F(vdivF, 1, 0, 0b11111) // Vd = Vn / Vm (float)
#undef F
#define F(mnemonic, U) \
void mnemonic(FloatRegister fd, FloatRegister fm, FloatRegister fn, \
int size, int quad) { \
assert(VM_Version::has_simd(), "simd instruction"); \
assert(!(size == VELEM_SIZE_64 && !quad), "reserved"); \
assert((size & 0b11) == size, "overflow"); \
int R = 0; /* rounding */ \
int S = 0; /* saturating */ \
emit_int32(quad << 30 | U << 29 | 0b01110 << 24 | size << 22 | \
1 << 21 | R << 12 | S << 11 | 0b10001 << 10 | \
fm->encoding() << 16 | \
fn->encoding() << 5 | \
fd->encoding()); \
}
F(vshlSI, 0) // Vd = ashift(Vn,Vm) (int)
F(vshlUI, 1) // Vd = lshift(Vn,Vm) (int)
#undef F
#define F(mnemonic, U, P, M) \
void mnemonic(FloatRegister fd, FloatRegister fn, FloatRegister fm, \
int size, int quad) { \
assert(VM_Version::has_simd(), "simd instruction"); \
assert(!(size == VELEM_SIZE_64 && !quad), "reserved"); \
assert(!(size == VELEM_SIZE_64 && M), "reserved"); \
assert((size & 0b11) == size, "overflow"); \
emit_int32(quad << 30 | U << 29 | 0b01110 << 24 | size << 22 | \
1 << 21 | P << 11 | 1 << 10 | \
fm->encoding() << 16 | \
fn->encoding() << 5 | \
fd->encoding()); \
}
F(vmulI, 0, 0b10011, true) // Vd = Vn * Vm (int)
F(vaddI, 0, 0b10000, false) // Vd = Vn + Vm (int)
F(vsubI, 1, 0b10000, false) // Vd = Vn - Vm (int)
#undef F
#define F(mnemonic, U, O) \
void mnemonic(FloatRegister fd, FloatRegister fn, FloatRegister fm, \
int quad) { \
assert(VM_Version::has_simd(), "simd instruction"); \
emit_int32(quad << 30 | U << 29 | 0b01110 << 24 | O << 22 | \
1 << 21 | 0b00011 << 11 | 1 << 10 | \
fm->encoding() << 16 | \
fn->encoding() << 5 | \
fd->encoding()); \
}
F(vandI, 0, 0b00) // Vd = Vn & Vm (int)
F(vorI, 0, 0b10) // Vd = Vn | Vm (int)
F(vxorI, 1, 0b00) // Vd = Vn ^ Vm (int)
#undef F
void vnegI(FloatRegister fd, FloatRegister fn, int size, int quad) {
int U = 1;
assert(VM_Version::has_simd(), "simd instruction");
assert(quad || size != VELEM_SIZE_64, "reserved");
emit_int32(quad << 30 | U << 29 | 0b01110 << 24 |
size << 22 | 0b100000101110 << 10 |
fn->encoding() << 5 |
fd->encoding() << 0);
}
void vshli(FloatRegister fd, FloatRegister fn, int esize, int imm, int quad) {
assert(VM_Version::has_simd(), "simd instruction");
if (imm >= esize) {
// maximum shift gives all zeroes, direction doesn't matter,
// but only available for shift right
vshri(fd, fn, esize, esize, true /* unsigned */, quad);
return;
}
assert(imm >= 0 && imm < esize, "out of range");
int imm7 = esize + imm;
int immh = imm7 >> 3;
assert(immh != 0, "encoding constraint");
assert((uint)immh < 16, "sanity");
assert(((immh >> 2) | quad) != 0b10, "reserved");
emit_int32(quad << 30 | 0b011110 << 23 | imm7 << 16 |
0b010101 << 10 | fn->encoding() << 5 | fd->encoding() << 0);
}
void vshri(FloatRegister fd, FloatRegister fn, int esize, int imm,
bool U /* unsigned */, int quad) {
assert(VM_Version::has_simd(), "simd instruction");
assert(imm > 0, "out of range");
if (imm >= esize) {
// maximum shift (all zeroes)
imm = esize;
}
int imm7 = 2 * esize - imm ;
int immh = imm7 >> 3;
assert(immh != 0, "encoding constraint");
assert((uint)immh < 16, "sanity");
assert(((immh >> 2) | quad) != 0b10, "reserved");
emit_int32(quad << 30 | U << 29 | 0b011110 << 23 | imm7 << 16 |
0b000001 << 10 | fn->encoding() << 5 | fd->encoding() << 0);
}
void vshrUI(FloatRegister fd, FloatRegister fm, int size, int imm, int quad) {
vshri(fd, fm, size, imm, true /* unsigned */, quad);
}
void vshrSI(FloatRegister fd, FloatRegister fm, int size, int imm, int quad) {
vshri(fd, fm, size, imm, false /* signed */, quad);
}
void vld1(FloatRegister Vt, Address addr, VElem_Size size, int bits) {
assert(VM_Version::has_simd(), "simd instruction");
assert(bits == 128, "unsupported");
assert(addr.disp() == 0 || addr.disp() == 16, "must be");
int type = 0b11; // 2D
int quad = 1;
int L = 1;
int opcode = VLD1_TYPE_1_REG;
emit_int32(quad << 30 | 0b11 << 26 | L << 22 | opcode << 12 | size << 10 |
Vt->encoding() << 0 | addr.encoding_simd());
}
void vst1(FloatRegister Vt, Address addr, VElem_Size size, int bits) {
assert(VM_Version::has_simd(), "simd instruction");
assert(bits == 128, "unsupported");
assert(addr.disp() == 0 || addr.disp() == 16, "must be");
int type = 0b11; // 2D
int quad = 1;
int L = 0;
int opcode = VLD1_TYPE_1_REG;
emit_int32(quad << 30 | 0b11 << 26 | L << 22 | opcode << 12 | size << 10 |
Vt->encoding() << 0 | addr.encoding_simd());
}
void vld1(FloatRegister Vt, FloatRegister Vt2, Address addr, VElem_Size size, int bits) {
assert(VM_Version::has_simd(), "simd instruction");
assert(bits == 128, "unsupported");
assert(Vt->successor() == Vt2, "Registers must be ordered");
assert(addr.disp() == 0 || addr.disp() == 32, "must be");
int type = 0b11; // 2D
int quad = 1;
int L = 1;
int opcode = VLD1_TYPE_2_REGS;
emit_int32(quad << 30 | 0b11 << 26 | L << 22 | opcode << 12 | size << 10 |
Vt->encoding() << 0 | addr.encoding_simd());
}
void vst1(FloatRegister Vt, FloatRegister Vt2, Address addr, VElem_Size size, int bits) {
assert(VM_Version::has_simd(), "simd instruction");
assert(Vt->successor() == Vt2, "Registers must be ordered");
assert(bits == 128, "unsupported");
assert(addr.disp() == 0 || addr.disp() == 32, "must be");
int type = 0b11; // 2D
int quad = 1;
int L = 0;
int opcode = VLD1_TYPE_2_REGS;
emit_int32(quad << 30 | 0b11 << 26 | L << 22 | opcode << 12 | size << 10 |
Vt->encoding() << 0 | addr.encoding_simd());
}
void vld1(FloatRegister Vt, FloatRegister Vt2, FloatRegister Vt3,
Address addr, VElem_Size size, int bits) {
assert(VM_Version::has_simd(), "simd instruction");
assert(bits == 128, "unsupported");
assert(Vt->successor() == Vt2 && Vt2->successor() == Vt3,
"Registers must be ordered");
assert(addr.disp() == 0 || addr.disp() == 48, "must be");
int type = 0b11; // 2D
int quad = 1;
int L = 1;
int opcode = VLD1_TYPE_3_REGS;
emit_int32(quad << 30 | 0b11 << 26 | L << 22 | opcode << 12 | size << 10 |
Vt->encoding() << 0 | addr.encoding_simd());
}
void vst1(FloatRegister Vt, FloatRegister Vt2, FloatRegister Vt3,
Address addr, VElem_Size size, int bits) {
assert(VM_Version::has_simd(), "simd instruction");
assert(bits == 128, "unsupported");
assert(Vt->successor() == Vt2 && Vt2->successor() == Vt3,
"Registers must be ordered");
assert(addr.disp() == 0 || addr.disp() == 48, "must be");
int type = 0b11; // 2D
int quad = 1;
int L = 0;
int opcode = VLD1_TYPE_3_REGS;
emit_int32(quad << 30 | 0b11 << 26 | L << 22 | opcode << 12 | size << 10 |
Vt->encoding() << 0 | addr.encoding_simd());
}
void vld1(FloatRegister Vt, FloatRegister Vt2, FloatRegister Vt3,
FloatRegister Vt4, Address addr, VElem_Size size, int bits) {
assert(VM_Version::has_simd(), "simd instruction");
assert(bits == 128, "unsupported");
assert(Vt->successor() == Vt2 && Vt2->successor() == Vt3 &&
Vt3->successor() == Vt4, "Registers must be ordered");
assert(addr.disp() == 0 || addr.disp() == 64, "must be");
int type = 0b11; // 2D
int quad = 1;
int L = 1;
int opcode = VLD1_TYPE_4_REGS;
emit_int32(quad << 30 | 0b11 << 26 | L << 22 | opcode << 12 | size << 10 |
Vt->encoding() << 0 | addr.encoding_simd());
}
void vst1(FloatRegister Vt, FloatRegister Vt2, FloatRegister Vt3,
FloatRegister Vt4, Address addr, VElem_Size size, int bits) {
assert(VM_Version::has_simd(), "simd instruction");
assert(bits == 128, "unsupported");
assert(Vt->successor() == Vt2 && Vt2->successor() == Vt3 &&
Vt3->successor() == Vt4, "Registers must be ordered");
assert(addr.disp() == 0 || addr.disp() == 64, "must be");
int type = 0b11; // 2D
int quad = 1;
int L = 0;
int opcode = VLD1_TYPE_4_REGS;
emit_int32(quad << 30 | 0b11 << 26 | L << 22 | opcode << 12 | size << 10 |
Vt->encoding() << 0 | addr.encoding_simd());
}
void rev32(FloatRegister Vd, FloatRegister Vn, VElem_Size size, int quad) {
assert(VM_Version::has_simd(), "simd instruction");
assert(size == VELEM_SIZE_8 || size == VELEM_SIZE_16, "must be");
emit_int32(quad << 30 | 0b101110 << 24 | size << 22 |
0b100000000010 << 10 | Vn->encoding() << 5 | Vd->encoding());
}
void eor(FloatRegister Vd, FloatRegister Vn, FloatRegister Vm, VElem_Size size, int quad) {
assert(VM_Version::has_simd(), "simd instruction");
assert(size == VELEM_SIZE_8, "must be");
emit_int32(quad << 30 | 0b101110001 << 21 | Vm->encoding() << 16 |
0b000111 << 10 | Vn->encoding() << 5 | Vd->encoding());
}
void orr(FloatRegister Vd, FloatRegister Vn, FloatRegister Vm, VElem_Size size, int quad) {
assert(VM_Version::has_simd(), "simd instruction");
assert(size == VELEM_SIZE_8, "must be");
emit_int32(quad << 30 | 0b001110101 << 21 | Vm->encoding() << 16 |
0b000111 << 10 | Vn->encoding() << 5 | Vd->encoding());
}
void vmovI(FloatRegister Dd, int imm8, VElem_Size size, int quad) {
assert(VM_Version::has_simd(), "simd instruction");
assert(imm8 >= 0 && imm8 < 256, "out of range");
int op;
int cmode;
switch (size) {
case VELEM_SIZE_8:
op = 0;
cmode = 0b1110;
break;
case VELEM_SIZE_16:
op = 0;
cmode = 0b1000;
break;
case VELEM_SIZE_32:
op = 0;
cmode = 0b0000;
break;
default:
cmode = 0;
ShouldNotReachHere();
}
int abc = imm8 >> 5;
int defgh = imm8 & 0b11111;
emit_int32(quad << 30 | op << 29 | 0b1111 << 24 |
abc << 16 | cmode << 12 | 0b01 << 10 |
defgh << 5 | Dd->encoding() << 0);
}
void vdupI(FloatRegister Dd, Register Rn, VElem_Size size, int quad) {
assert(VM_Version::has_simd(), "simd instruction");
assert(size <= 3, "unallocated encoding");
assert(size != 3 || quad == 1, "reserved");
int imm5 = 1 << size;
#ifdef ASSERT
switch (size) {
case VELEM_SIZE_8:
assert(imm5 == 0b00001, "sanity");
break;
case VELEM_SIZE_16:
assert(imm5 == 0b00010, "sanity");
break;
case VELEM_SIZE_32:
assert(imm5 == 0b00100, "sanity");
break;
case VELEM_SIZE_64:
assert(imm5 == 0b01000, "sanity");
break;
default:
ShouldNotReachHere();
}
#endif
emit_int32(quad << 30 | 0b111 << 25 | 0b11 << 10 |
imm5 << 16 | Rn->encoding() << 5 |
Dd->encoding() << 0);
}
void vdup(FloatRegister Vd, FloatRegister Vn, VElem_Size size, int quad) {
assert(VM_Version::has_simd(), "simd instruction");
int index = 0;
int bytes = 1 << size;
int range = 16 / bytes;
assert(index < range, "overflow");
assert(size != VELEM_SIZE_64 || quad, "reserved");
assert(8 << VELEM_SIZE_8 == 8, "sanity");
assert(8 << VELEM_SIZE_16 == 16, "sanity");
assert(8 << VELEM_SIZE_32 == 32, "sanity");
assert(8 << VELEM_SIZE_64 == 64, "sanity");
int imm5 = (index << (size + 1)) | bytes;
emit_int32(quad << 30 | 0b001110000 << 21 | imm5 << 16 | 0b000001 << 10 |
Vn->encoding() << 5 | Vd->encoding() << 0);
}
void vdupF(FloatRegister Vd, FloatRegister Vn, int quad) {
vdup(Vd, Vn, VELEM_SIZE_32, quad);
}
void vdupD(FloatRegister Vd, FloatRegister Vn, int quad) {
vdup(Vd, Vn, VELEM_SIZE_64, quad);
}
#endif
};
#endif // CPU_ARM_VM_ASSEMBLER_ARM_64_HPP