jdk-sandbox: comparison hotspot/src/cpu/arm/vm/arm.ad

equal deleted inserted replaced

-:2335df372367
+:29142a56c193
+//
+// Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved.
+// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+//
+// This code is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License version 2 only, as
+// published by the Free Software Foundation.
+//
+// This code is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// version 2 for more details (a copy is included in the LICENSE file that
+// accompanied this code).
+//
+// You should have received a copy of the GNU General Public License version
+// 2 along with this work; if not, write to the Free Software Foundation,
+// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+// or visit www.oracle.com if you need additional information or have any
+// questions.
+//
+// ARM Architecture Description File
+//----------DEFINITION BLOCK---------------------------------------------------
+// Define name --> value mappings to inform the ADLC of an integer valued name
+// Current support includes integer values in the range [0, 0x7FFFFFFF]
+// Format:
+//        int_def  <name>         ( <int_value>, <expression>);
+// Generated Code in ad_<arch>.hpp
+//        #define  <name>   (<expression>)
+//        // value == <int_value>
+// Generated code in ad_<arch>.cpp adlc_verification()
+//        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
+//
+definitions %{
+// The default cost (of an ALU instruction).
+int_def DEFAULT_COST      (    100,     100);
+int_def HUGE_COST         (1000000, 1000000);
+// Memory refs are twice as expensive as run-of-the-mill.
+int_def MEMORY_REF_COST   (    200, DEFAULT_COST * 2);
+// Branches are even more expensive.
+int_def BRANCH_COST       (    300, DEFAULT_COST * 3);
+int_def CALL_COST         (    300, DEFAULT_COST * 3);
+%}
+//----------SOURCE BLOCK-------------------------------------------------------
+// This is a block of C++ code which provides values, functions, and
+// definitions necessary in the rest of the architecture description
+source_hpp %{
+// Header information of the source block.
+// Method declarations/definitions which are used outside
+// the ad-scope can conveniently be defined here.
+//
+// To keep related declarations/definitions/uses close together,
+// we switch between source %{ }% and source_hpp %{ }% freely as needed.
+// Does destination need to be loaded in a register then passed to a
+// branch instruction?
+extern bool maybe_far_call(const CallNode *n);
+extern bool maybe_far_call(const MachCallNode *n);
+static inline bool cache_reachable() {
+return MacroAssembler::_cache_fully_reachable();
+}
+#ifdef AARCH64
+#define ldr_32 ldr_w
+#define str_32 str_w
+#else
+#define ldr_32 ldr
+#define str_32 str
+#define tst_32 tst
+#define teq_32 teq
+#endif
+#if 1
+extern bool PrintOptoAssembly;
+#endif
+class c2 {
+public:
+static OptoRegPair return_value(int ideal_reg);
+};
+class CallStubImpl {
+//--------------------------------------------------------------
+//---<  Used for optimization in Compile::Shorten_branches  >---
+//--------------------------------------------------------------
+public:
+// Size of call trampoline stub.
+static uint size_call_trampoline() {
+return 0; // no call trampolines on this platform
+}
+// number of relocations needed by a call trampoline stub
+static uint reloc_call_trampoline() {
+return 0; // no call trampolines on this platform
+}
+};
+class HandlerImpl {
+public:
+static int emit_exception_handler(CodeBuffer &cbuf);
+static int emit_deopt_handler(CodeBuffer& cbuf);
+static uint size_exception_handler() {
+#ifdef AARCH64
+// ldr_literal; br; (pad); <literal>
+return 3 * Assembler::InstructionSize + wordSize;
+#else
+return ( 3 * 4 );
+#endif
+}
+static uint size_deopt_handler() {
+return ( 9 * 4 );
+}
+};
+%}
+source %{
+#define __ _masm.
+static FloatRegister reg_to_FloatRegister_object(int register_encoding);
+static Register reg_to_register_object(int register_encoding);
+// ****************************************************************************
+// REQUIRED FUNCTIONALITY
+// Indicate if the safepoint node needs the polling page as an input.
+// Since ARM does not have absolute addressing, it does.
+bool SafePointNode::needs_polling_address_input() {
+return true;
+}
+// emit an interrupt that is caught by the debugger (for debugging compiler)
+void emit_break(CodeBuffer &cbuf) {
+MacroAssembler _masm(&cbuf);
+__ breakpoint();
+}
+#ifndef PRODUCT
+void MachBreakpointNode::format( PhaseRegAlloc *, outputStream *st ) const {
+st->print("TA");
+}
+#endif
+void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
+emit_break(cbuf);
+}
+uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
+return MachNode::size(ra_);
+}
+void emit_nop(CodeBuffer &cbuf) {
+MacroAssembler _masm(&cbuf);
+__ nop();
+}
+void emit_call_reloc(CodeBuffer &cbuf, const MachCallNode *n, MachOper *m, RelocationHolder const& rspec) {
+int ret_addr_offset0 = n->as_MachCall()->ret_addr_offset();
+int call_site_offset = cbuf.insts()->mark_off();
+MacroAssembler _masm(&cbuf);
+__ set_inst_mark(); // needed in emit_to_interp_stub() to locate the call
+address target = (address)m->method();
+assert(n->as_MachCall()->entry_point() == target, "sanity");
+assert(maybe_far_call(n) == !__ reachable_from_cache(target), "sanity");
+assert(cache_reachable() == __ cache_fully_reachable(), "sanity");
+assert(target != NULL, "need real address");
+int ret_addr_offset = -1;
+if (rspec.type() == relocInfo::runtime_call_type) {
+__ call(target, rspec);
+ret_addr_offset = __ offset();
+} else {
+// scratches Rtemp
+ret_addr_offset = __ patchable_call(target, rspec, true);
+}
+assert(ret_addr_offset - call_site_offset == ret_addr_offset0, "fix ret_addr_offset()");
+}
+//=============================================================================
+// REQUIRED FUNCTIONALITY for encoding
+void emit_lo(CodeBuffer &cbuf, int val) {  }
+void emit_hi(CodeBuffer &cbuf, int val) {  }
+//=============================================================================
+const RegMask& MachConstantBaseNode::_out_RegMask = PTR_REG_mask();
+int Compile::ConstantTable::calculate_table_base_offset() const {
+#ifdef AARCH64
+return 0;
+#else
+int offset = -(size() / 2);
+// flds, fldd: 8-bit  offset multiplied by 4: +/- 1024
+// ldr, ldrb : 12-bit offset:                 +/- 4096
+if (!Assembler::is_simm10(offset)) {
+offset = Assembler::min_simm10();
+}
+return offset;
+#endif
+}
+bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
+void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
+ShouldNotReachHere();
+}
+void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
+Compile* C = ra_->C;
+Compile::ConstantTable& constant_table = C->constant_table();
+MacroAssembler _masm(&cbuf);
+Register r = as_Register(ra_->get_encode(this));
+CodeSection* consts_section = __ code()->consts();
+int consts_size = consts_section->align_at_start(consts_section->size());
+assert(constant_table.size() == consts_size, "must be: %d == %d", constant_table.size(), consts_size);
+// Materialize the constant table base.
+address baseaddr = consts_section->start() + -(constant_table.table_base_offset());
+RelocationHolder rspec = internal_word_Relocation::spec(baseaddr);
+__ mov_address(r, baseaddr, rspec);
+}
+uint MachConstantBaseNode::size(PhaseRegAlloc*) const {
+#ifdef AARCH64
+return 5 * Assembler::InstructionSize;
+#else
+return 8;
+#endif
+}
+#ifndef PRODUCT
+void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
+char reg[128];
+ra_->dump_register(this, reg);
+st->print("MOV_SLOW    &constanttable,%s\t! constant table base", reg);
+}
+#endif
+#ifndef PRODUCT
+void MachPrologNode::format( PhaseRegAlloc *ra_, outputStream *st ) const {
+Compile* C = ra_->C;
+for (int i = 0; i < OptoPrologueNops; i++) {
+st->print_cr("NOP"); st->print("\t");
+}
+#ifdef AARCH64
+if (OptoPrologueNops <= 0) {
+st->print_cr("NOP\t! required for safe patching");
+st->print("\t");
+}
+#endif
+size_t framesize = C->frame_size_in_bytes();
+assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
+int bangsize = C->bang_size_in_bytes();
+// Remove two words for return addr and rbp,
+framesize -= 2*wordSize;
+bangsize -= 2*wordSize;
+// Calls to C2R adapters often do not accept exceptional returns.
+// We require that their callers must bang for them.  But be careful, because
+// some VM calls (such as call site linkage) can use several kilobytes of
+// stack.  But the stack safety zone should account for that.
+// See bugs 4446381, 4468289, 4497237.
+if (C->need_stack_bang(bangsize)) {
+st->print_cr("! stack bang (%d bytes)", bangsize); st->print("\t");
+}
+st->print_cr("PUSH   R_FP|R_LR_LR"); st->print("\t");
+if (framesize != 0) {
+st->print   ("SUB    R_SP, R_SP, " SIZE_FORMAT,framesize);
+}
+}
+#endif
+void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
+Compile* C = ra_->C;
+MacroAssembler _masm(&cbuf);
+for (int i = 0; i < OptoPrologueNops; i++) {
+__ nop();
+}
+#ifdef AARCH64
+if (OptoPrologueNops <= 0) {
+__ nop(); // required for safe patching by patch_verified_entry()
+}
+#endif
+size_t framesize = C->frame_size_in_bytes();
+assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
+int bangsize = C->bang_size_in_bytes();
+// Remove two words for return addr and fp,
+framesize -= 2*wordSize;
+bangsize -= 2*wordSize;
+// Calls to C2R adapters often do not accept exceptional returns.
+// We require that their callers must bang for them.  But be careful, because
+// some VM calls (such as call site linkage) can use several kilobytes of
+// stack.  But the stack safety zone should account for that.
+// See bugs 4446381, 4468289, 4497237.
+if (C->need_stack_bang(bangsize)) {
+__ arm_stack_overflow_check(bangsize, Rtemp);
+}
+__ raw_push(FP, LR);
+if (framesize != 0) {
+__ sub_slow(SP, SP, framesize);
+}
+// offset from scratch buffer is not valid
+if (strcmp(cbuf.name(), "Compile::Fill_buffer") == 0) {
+C->set_frame_complete( __ offset() );
+}
+if (C->has_mach_constant_base_node()) {
+// NOTE: We set the table base offset here because users might be
+// emitted before MachConstantBaseNode.
+Compile::ConstantTable& constant_table = C->constant_table();
+constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
+}
+}
+uint MachPrologNode::size(PhaseRegAlloc *ra_) const {
+return MachNode::size(ra_);
+}
+int MachPrologNode::reloc() const {
+return 10; // a large enough number
+}
+//=============================================================================
+#ifndef PRODUCT
+void MachEpilogNode::format( PhaseRegAlloc *ra_, outputStream *st ) const {
+Compile* C = ra_->C;
+size_t framesize = C->frame_size_in_bytes();
+framesize -= 2*wordSize;
+if (framesize != 0) {
+st->print("ADD    R_SP, R_SP, " SIZE_FORMAT "\n\t",framesize);
+}
+st->print("POP    R_FP|R_LR_LR");
+if (do_polling() && ra_->C->is_method_compilation()) {
+st->print("\n\t");
+#ifdef AARCH64
+if (MacroAssembler::page_reachable_from_cache(os::get_polling_page())) {
+st->print("ADRP     Rtemp, #PollAddr\t! Load Polling address\n\t");
+st->print("LDR      ZR,[Rtemp + #PollAddr & 0xfff]\t!Poll for Safepointing");
+} else {
+st->print("mov_slow Rtemp, #PollAddr\t! Load Polling address\n\t");
+st->print("LDR      ZR,[Rtemp]\t!Poll for Safepointing");
+}
+#else
+st->print("MOV    Rtemp, #PollAddr\t! Load Polling address\n\t");
+st->print("LDR    Rtemp,[Rtemp]\t!Poll for Safepointing");
+#endif
+}
+}
+#endif
+void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
+MacroAssembler _masm(&cbuf);
+Compile* C = ra_->C;
+size_t framesize = C->frame_size_in_bytes();
+framesize -= 2*wordSize;
+if (framesize != 0) {
+__ add_slow(SP, SP, framesize);
+}
+__ raw_pop(FP, LR);
+// If this does safepoint polling, then do it here
+if (do_polling() && ra_->C->is_method_compilation()) {
+#ifdef AARCH64
+if (false && MacroAssembler::page_reachable_from_cache(os::get_polling_page())) {
+/* FIXME: TODO
+__ relocate(relocInfo::xxx);
+__ adrp(Rtemp, (intptr_t)os::get_polling_page());
+__ relocate(relocInfo::poll_return_type);
+int offset = os::get_polling_page() & 0xfff;
+__ ldr(ZR, Address(Rtemp + offset));
+*/
+} else {
+__ mov_address(Rtemp, (address)os::get_polling_page(), symbolic_Relocation::polling_page_reference);
+__ relocate(relocInfo::poll_return_type);
+__ ldr(ZR, Address(Rtemp));
+}
+#else
+// mov_slow here is usually one or two instruction
+__ mov_address(Rtemp, (address)os::get_polling_page(), symbolic_Relocation::polling_page_reference);
+__ relocate(relocInfo::poll_return_type);
+__ ldr(Rtemp, Address(Rtemp));
+#endif
+}
+}
+uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
+#ifdef AARCH64
+// allow for added alignment nop from mov_address bind_literal
+return MachNode::size(ra_) + 1 * Assembler::InstructionSize;
+#else
+return MachNode::size(ra_);
+#endif
+}
+int MachEpilogNode::reloc() const {
+return 16; // a large enough number
+}
+const Pipeline * MachEpilogNode::pipeline() const {
+return MachNode::pipeline_class();
+}
+int MachEpilogNode::safepoint_offset() const {
+assert( do_polling(), "no return for this epilog node");
+//  return MacroAssembler::size_of_sethi(os::get_polling_page());
+Unimplemented();
+return 0;
+}
+//=============================================================================
+// Figure out which register class each belongs in: rc_int, rc_float, rc_stack
+enum RC { rc_bad, rc_int, rc_float, rc_stack };
+static enum RC rc_class( OptoReg::Name reg ) {
+if (!OptoReg::is_valid(reg)) return rc_bad;
+if (OptoReg::is_stack(reg)) return rc_stack;
+VMReg r = OptoReg::as_VMReg(reg);
+if (r->is_Register()) return rc_int;
+assert(r->is_FloatRegister(), "must be");
+return rc_float;
+}
+static inline bool is_iRegLd_memhd(OptoReg::Name src_first, OptoReg::Name src_second, int offset) {
+#ifdef AARCH64
+return is_memoryHD(offset);
+#else
+int rlo = Matcher::_regEncode[src_first];
+int rhi = Matcher::_regEncode[src_second];
+if (!((rlo&1)==0 && (rlo+1 == rhi))) {
+tty->print_cr("CAUGHT BAD LDRD/STRD");
+}
+return (rlo&1)==0 && (rlo+1 == rhi) && is_memoryHD(offset);
+#endif
+}
+uint MachSpillCopyNode::implementation( CodeBuffer *cbuf,
+PhaseRegAlloc *ra_,
+bool do_size,
+outputStream* st ) const {
+// Get registers to move
+OptoReg::Name src_second = ra_->get_reg_second(in(1));
+OptoReg::Name src_first = ra_->get_reg_first(in(1));
+OptoReg::Name dst_second = ra_->get_reg_second(this );
+OptoReg::Name dst_first = ra_->get_reg_first(this );
+enum RC src_second_rc = rc_class(src_second);
+enum RC src_first_rc = rc_class(src_first);
+enum RC dst_second_rc = rc_class(dst_second);
+enum RC dst_first_rc = rc_class(dst_first);
+assert( OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first), "must move at least 1 register" );
+// Generate spill code!
+int size = 0;
+if (src_first == dst_first && src_second == dst_second)
+return size;            // Self copy, no move
+#ifdef TODO
+if (bottom_type()->isa_vect() != NULL) {
+}
+#endif
+// Shared code does not expect instruction set capability based bailouts here.
+// Handle offset unreachable bailout with minimal change in shared code.
+// Bailout only for real instruction emit.
+// This requires a single comment change in shared code. ( see output.cpp "Normal" instruction case )
+MacroAssembler _masm(cbuf);
+// --------------------------------------
+// Check for mem-mem move.  Load into unused float registers and fall into
+// the float-store case.
+if (src_first_rc == rc_stack && dst_first_rc == rc_stack) {
+int offset = ra_->reg2offset(src_first);
+if (cbuf && !is_memoryfp(offset)) {
+ra_->C->record_method_not_compilable("unable to handle large constant offsets");
+return 0;
+} else {
+if (src_second_rc != rc_bad) {
+assert((src_first&1)==0 && src_first+1 == src_second, "pair of registers must be aligned/contiguous");
+src_first     = OptoReg::Name(R_mem_copy_lo_num);
+src_second    = OptoReg::Name(R_mem_copy_hi_num);
+src_first_rc  = rc_float;
+src_second_rc = rc_float;
+if (cbuf) {
+__ ldr_double(Rmemcopy, Address(SP, offset));
+} else if (!do_size) {
+st->print(LDR_DOUBLE "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(src_first),offset);
+}
+} else {
+src_first     = OptoReg::Name(R_mem_copy_lo_num);
+src_first_rc  = rc_float;
+if (cbuf) {
+__ ldr_float(Rmemcopy, Address(SP, offset));
+} else if (!do_size) {
+st->print(LDR_FLOAT "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(src_first),offset);
+}
+}
+size += 4;
+}
+}
+if (src_second_rc == rc_stack && dst_second_rc == rc_stack) {
+Unimplemented();
+}
+// --------------------------------------
+// Check for integer reg-reg copy
+if (src_first_rc == rc_int && dst_first_rc == rc_int) {
+// Else normal reg-reg copy
+assert( src_second != dst_first, "smashed second before evacuating it" );
+if (cbuf) {
+__ mov(reg_to_register_object(Matcher::_regEncode[dst_first]), reg_to_register_object(Matcher::_regEncode[src_first]));
+#ifndef PRODUCT
+} else if (!do_size) {
+st->print("MOV    R_%s, R_%s\t# spill",
+Matcher::regName[dst_first],
+Matcher::regName[src_first]);
+#endif
+}
+#ifdef AARCH64
+if (src_first+1 == src_second && dst_first+1 == dst_second) {
+return size + 4;
+}
+#endif
+size += 4;
+}
+// Check for integer store
+if (src_first_rc == rc_int && dst_first_rc == rc_stack) {
+int offset = ra_->reg2offset(dst_first);
+if (cbuf && !is_memoryI(offset)) {
+ra_->C->record_method_not_compilable("unable to handle large constant offsets");
+return 0;
+} else {
+if (src_second_rc != rc_bad && is_iRegLd_memhd(src_first, src_second, offset)) {
+assert((src_first&1)==0 && src_first+1 == src_second, "pair of registers must be aligned/contiguous");
+if (cbuf) {
+__ str_64(reg_to_register_object(Matcher::_regEncode[src_first]), Address(SP, offset));
+#ifndef PRODUCT
+} else if (!do_size) {
+if (size != 0) st->print("\n\t");
+st->print(STR_64 "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(src_first), offset);
+#endif
+}
+return size + 4;
+} else {
+if (cbuf) {
+__ str_32(reg_to_register_object(Matcher::_regEncode[src_first]), Address(SP, offset));
+#ifndef PRODUCT
+} else if (!do_size) {
+if (size != 0) st->print("\n\t");
+st->print(STR_32 "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(src_first), offset);
+#endif
+}
+}
+}
+size += 4;
+}
+// Check for integer load
+if (dst_first_rc == rc_int && src_first_rc == rc_stack) {
+int offset = ra_->reg2offset(src_first);
+if (cbuf && !is_memoryI(offset)) {
+ra_->C->record_method_not_compilable("unable to handle large constant offsets");
+return 0;
+} else {
+if (src_second_rc != rc_bad && is_iRegLd_memhd(dst_first, dst_second, offset)) {
+assert((src_first&1)==0 && src_first+1 == src_second, "pair of registers must be aligned/contiguous");
+if (cbuf) {
+__ ldr_64(reg_to_register_object(Matcher::_regEncode[dst_first]), Address(SP, offset));
+#ifndef PRODUCT
+} else if (!do_size) {
+if (size != 0) st->print("\n\t");
+st->print(LDR_64 "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(dst_first), offset);
+#endif
+}
+return size + 4;
+} else {
+if (cbuf) {
+__ ldr_32(reg_to_register_object(Matcher::_regEncode[dst_first]), Address(SP, offset));
+#ifndef PRODUCT
+} else if (!do_size) {
+if (size != 0) st->print("\n\t");
+st->print(LDR_32 "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(dst_first), offset);
+#endif
+}
+}
+}
+size += 4;
+}
+// Check for float reg-reg copy
+if (src_first_rc == rc_float && dst_first_rc == rc_float) {
+if (src_second_rc != rc_bad) {
+assert((src_first&1)==0 && src_first+1 == src_second && (dst_first&1)==0 && dst_first+1 == dst_second, "pairs of registers must be aligned/contiguous");
+if (cbuf) {
+__ mov_double(reg_to_FloatRegister_object(Matcher::_regEncode[dst_first]), reg_to_FloatRegister_object(Matcher::_regEncode[src_first]));
+#ifndef PRODUCT
+} else if (!do_size) {
+st->print(MOV_DOUBLE "    R_%s, R_%s\t# spill",
+Matcher::regName[dst_first],
+Matcher::regName[src_first]);
+#endif
+}
+return 4;
+}
+if (cbuf) {
+__ mov_float(reg_to_FloatRegister_object(Matcher::_regEncode[dst_first]), reg_to_FloatRegister_object(Matcher::_regEncode[src_first]));
+#ifndef PRODUCT
+} else if (!do_size) {
+st->print(MOV_FLOAT "    R_%s, R_%s\t# spill",
+Matcher::regName[dst_first],
+Matcher::regName[src_first]);
+#endif
+}
+size = 4;
+}
+// Check for float store
+if (src_first_rc == rc_float && dst_first_rc == rc_stack) {
+int offset = ra_->reg2offset(dst_first);
+if (cbuf && !is_memoryfp(offset)) {
+ra_->C->record_method_not_compilable("unable to handle large constant offsets");
+return 0;
+} else {
+// Further check for aligned-adjacent pair, so we can use a double store
+if (src_second_rc != rc_bad) {
+assert((src_first&1)==0 && src_first+1 == src_second && (dst_first&1)==0 && dst_first+1 == dst_second, "pairs of registers and stack slots must be aligned/contiguous");
+if (cbuf) {
+__ str_double(reg_to_FloatRegister_object(Matcher::_regEncode[src_first]), Address(SP, offset));
+#ifndef PRODUCT
+} else if (!do_size) {
+if (size != 0) st->print("\n\t");
+st->print(STR_DOUBLE "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(src_first),offset);
+#endif
+}
+return size + 4;
+} else {
+if (cbuf) {
+__ str_float(reg_to_FloatRegister_object(Matcher::_regEncode[src_first]), Address(SP, offset));
+#ifndef PRODUCT
+} else if (!do_size) {
+if (size != 0) st->print("\n\t");
+st->print(STR_FLOAT "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(src_first),offset);
+#endif
+}
+}
+}
+size += 4;
+}
+// Check for float load
+if (dst_first_rc == rc_float && src_first_rc == rc_stack) {
+int offset = ra_->reg2offset(src_first);
+if (cbuf && !is_memoryfp(offset)) {
+ra_->C->record_method_not_compilable("unable to handle large constant offsets");
+return 0;
+} else {
+// Further check for aligned-adjacent pair, so we can use a double store
+if (src_second_rc != rc_bad) {
+assert((src_first&1)==0 && src_first+1 == src_second && (dst_first&1)==0 && dst_first+1 == dst_second, "pairs of registers and stack slots must be aligned/contiguous");
+if (cbuf) {
+__ ldr_double(reg_to_FloatRegister_object(Matcher::_regEncode[dst_first]), Address(SP, offset));
+#ifndef PRODUCT
+} else if (!do_size) {
+if (size != 0) st->print("\n\t");
+st->print(LDR_DOUBLE "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(dst_first),offset);
+#endif
+}
+return size + 4;
+} else {
+if (cbuf) {
+__ ldr_float(reg_to_FloatRegister_object(Matcher::_regEncode[dst_first]), Address(SP, offset));
+#ifndef PRODUCT
+} else if (!do_size) {
+if (size != 0) st->print("\n\t");
+st->print(LDR_FLOAT "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(dst_first),offset);
+#endif
+}
+}
+}
+size += 4;
+}
+// check for int reg -> float reg move
+if (src_first_rc == rc_int && dst_first_rc == rc_float) {
+// Further check for aligned-adjacent pair, so we can use a single instruction
+if (src_second_rc != rc_bad) {
+assert((dst_first&1)==0 && dst_first+1 == dst_second, "pairs of registers must be aligned/contiguous");
+assert((src_first&1)==0 && src_first+1 == src_second, "pairs of registers must be aligned/contiguous");
+assert(src_second_rc == rc_int && dst_second_rc == rc_float, "unsupported");
+if (cbuf) {
+#ifdef AARCH64
+__ fmov_dx(reg_to_FloatRegister_object(Matcher::_regEncode[dst_first]), reg_to_register_object(Matcher::_regEncode[src_first]));
+#else
+__ fmdrr(reg_to_FloatRegister_object(Matcher::_regEncode[dst_first]), reg_to_register_object(Matcher::_regEncode[src_first]), reg_to_register_object(Matcher::_regEncode[src_second]));
+#endif
+#ifndef PRODUCT
+} else if (!do_size) {
+if (size != 0) st->print("\n\t");
+#ifdef AARCH64
+st->print("FMOV_DX   R_%s, R_%s\t! spill",OptoReg::regname(dst_first), OptoReg::regname(src_first));
+#else
+st->print("FMDRR   R_%s, R_%s, R_%s\t! spill",OptoReg::regname(dst_first), OptoReg::regname(src_first), OptoReg::regname(src_second));
+#endif
+#endif
+}
+return size + 4;
+} else {
+if (cbuf) {
+__ fmsr(reg_to_FloatRegister_object(Matcher::_regEncode[dst_first]), reg_to_register_object(Matcher::_regEncode[src_first]));
+#ifndef PRODUCT
+} else if (!do_size) {
+if (size != 0) st->print("\n\t");
+st->print(FMSR "   R_%s, R_%s\t! spill",OptoReg::regname(dst_first), OptoReg::regname(src_first));
+#endif
+}
+size += 4;
+}
+}
+// check for float reg -> int reg move
+if (src_first_rc == rc_float && dst_first_rc == rc_int) {
+// Further check for aligned-adjacent pair, so we can use a single instruction
+if (src_second_rc != rc_bad) {
+assert((src_first&1)==0 && src_first+1 == src_second, "pairs of registers must be aligned/contiguous");
+assert((dst_first&1)==0 && dst_first+1 == dst_second, "pairs of registers must be aligned/contiguous");
+assert(src_second_rc == rc_float && dst_second_rc == rc_int, "unsupported");
+if (cbuf) {
+#ifdef AARCH64
+__ fmov_xd(reg_to_register_object(Matcher::_regEncode[dst_first]), reg_to_FloatRegister_object(Matcher::_regEncode[src_first]));
+#else
+__ fmrrd(reg_to_register_object(Matcher::_regEncode[dst_first]), reg_to_register_object(Matcher::_regEncode[dst_second]), reg_to_FloatRegister_object(Matcher::_regEncode[src_first]));
+#endif
+#ifndef PRODUCT
+} else if (!do_size) {
+if (size != 0) st->print("\n\t");
+#ifdef AARCH64
+st->print("FMOV_XD R_%s, R_%s\t! spill",OptoReg::regname(dst_first), OptoReg::regname(src_first));
+#else
+st->print("FMRRD   R_%s, R_%s, R_%s\t! spill",OptoReg::regname(dst_first), OptoReg::regname(dst_second), OptoReg::regname(src_first));
+#endif
+#endif
+}
+return size + 4;
+} else {
+if (cbuf) {
+__ fmrs(reg_to_register_object(Matcher::_regEncode[dst_first]), reg_to_FloatRegister_object(Matcher::_regEncode[src_first]));
+#ifndef PRODUCT
+} else if (!do_size) {
+if (size != 0) st->print("\n\t");
+st->print(FMRS "   R_%s, R_%s\t! spill",OptoReg::regname(dst_first), OptoReg::regname(src_first));
+#endif
+}
+size += 4;
+}
+}
+// --------------------------------------------------------------------
+// Check for hi bits still needing moving.  Only happens for misaligned
+// arguments to native calls.
+if (src_second == dst_second)
+return size;               // Self copy; no move
+assert( src_second_rc != rc_bad && dst_second_rc != rc_bad, "src_second & dst_second cannot be Bad" );
+#ifndef AARCH64
+// Check for integer reg-reg copy.  Hi bits are stuck up in the top
+// 32-bits of a 64-bit register, but are needed in low bits of another
+// register (else it's a hi-bits-to-hi-bits copy which should have
+// happened already as part of a 64-bit move)
+if (src_second_rc == rc_int && dst_second_rc == rc_int) {
+if (cbuf) {
+__ mov(reg_to_register_object(Matcher::_regEncode[dst_second]), reg_to_register_object(Matcher::_regEncode[src_second]));
+#ifndef PRODUCT
+} else if (!do_size) {
+if (size != 0) st->print("\n\t");
+st->print("MOV    R_%s, R_%s\t# spill high",
+Matcher::regName[dst_second],
+Matcher::regName[src_second]);
+#endif
+}
+return size+4;
+}
+// Check for high word integer store
+if (src_second_rc == rc_int && dst_second_rc == rc_stack) {
+int offset = ra_->reg2offset(dst_second);
+if (cbuf && !is_memoryP(offset)) {
+ra_->C->record_method_not_compilable("unable to handle large constant offsets");
+return 0;
+} else {
+if (cbuf) {
+__ str(reg_to_register_object(Matcher::_regEncode[src_second]), Address(SP, offset));
+#ifndef PRODUCT
+} else if (!do_size) {
+if (size != 0) st->print("\n\t");
+st->print("STR   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(src_second), offset);
+#endif
+}
+}
+return size + 4;
+}
+// Check for high word integer load
+if (dst_second_rc == rc_int && src_second_rc == rc_stack) {
+int offset = ra_->reg2offset(src_second);
+if (cbuf && !is_memoryP(offset)) {
+ra_->C->record_method_not_compilable("unable to handle large constant offsets");
+return 0;
+} else {
+if (cbuf) {
+__ ldr(reg_to_register_object(Matcher::_regEncode[dst_second]), Address(SP, offset));
+#ifndef PRODUCT
+} else if (!do_size) {
+if (size != 0) st->print("\n\t");
+st->print("LDR   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(dst_second), offset);
+#endif
+}
+}
+return size + 4;
+}
+#endif
+Unimplemented();
+return 0; // Mute compiler
+}
+#ifndef PRODUCT
+void MachSpillCopyNode::format( PhaseRegAlloc *ra_, outputStream *st ) const {
+implementation( NULL, ra_, false, st );
+}
+#endif
+void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
+implementation( &cbuf, ra_, false, NULL );
+}
+uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
+return implementation( NULL, ra_, true, NULL );
+}
+//=============================================================================
+#ifndef PRODUCT
+void MachNopNode::format( PhaseRegAlloc *, outputStream *st ) const {
+st->print("NOP \t# %d bytes pad for loops and calls", 4 * _count);
+}
+#endif
+void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc * ) const {
+MacroAssembler _masm(&cbuf);
+for(int i = 0; i < _count; i += 1) {
+__ nop();
+}
+}
+uint MachNopNode::size(PhaseRegAlloc *ra_) const {
+return 4 * _count;
+}
+//=============================================================================
+#ifndef PRODUCT
+void BoxLockNode::format( PhaseRegAlloc *ra_, outputStream *st ) const {
+int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
+int reg = ra_->get_reg_first(this);
+st->print("ADD    %s,R_SP+#%d",Matcher::regName[reg], offset);
+}
+#endif
+void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
+MacroAssembler _masm(&cbuf);
+int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
+int reg = ra_->get_encode(this);
+Register dst = reg_to_register_object(reg);
+if (is_aimm(offset)) {
+__ add(dst, SP, offset);
+} else {
+__ mov_slow(dst, offset);
+#ifdef AARCH64
+__ add(dst, SP, dst, ex_lsl);
+#else
+__ add(dst, SP, dst);
+#endif
+}
+}
+uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
+// BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_)
+assert(ra_ == ra_->C->regalloc(), "sanity");
+return ra_->C->scratch_emit_size(this);
+}
+//=============================================================================
+#ifndef PRODUCT
+#ifdef AARCH64
+#define R_RTEMP "R_R16"
+#else
+#define R_RTEMP "R_R12"
+#endif
+void MachUEPNode::format( PhaseRegAlloc *ra_, outputStream *st ) const {
+st->print_cr("\nUEP:");
+if (UseCompressedClassPointers) {
+st->print_cr("\tLDR_w " R_RTEMP ",[R_R0 + oopDesc::klass_offset_in_bytes]\t! Inline cache check");
+st->print_cr("\tdecode_klass " R_RTEMP);
+} else {
+st->print_cr("\tLDR   " R_RTEMP ",[R_R0 + oopDesc::klass_offset_in_bytes]\t! Inline cache check");
+}
+st->print_cr("\tCMP   " R_RTEMP ",R_R8" );
+st->print   ("\tB.NE  SharedRuntime::handle_ic_miss_stub");
+}
+#endif
+void MachUEPNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
+MacroAssembler _masm(&cbuf);
+Register iCache  = reg_to_register_object(Matcher::inline_cache_reg_encode());
+assert(iCache == Ricklass, "should be");
+Register receiver = R0;
+__ load_klass(Rtemp, receiver);
+__ cmp(Rtemp, iCache);
+#ifdef AARCH64
+Label match;
+__ b(match, eq);
+__ jump(SharedRuntime::get_ic_miss_stub(), relocInfo::runtime_call_type, Rtemp);
+__ bind(match);
+#else
+__ jump(SharedRuntime::get_ic_miss_stub(), relocInfo::runtime_call_type, noreg, ne);
+#endif
+}
+uint MachUEPNode::size(PhaseRegAlloc *ra_) const {
+return MachNode::size(ra_);
+}
+//=============================================================================
+// Emit exception handler code.
+int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf) {
+MacroAssembler _masm(&cbuf);
+address base = __ start_a_stub(size_exception_handler());
+if (base == NULL) {
+ciEnv::current()->record_failure("CodeCache is full");
+return 0;  // CodeBuffer::expand failed
+}
+int offset = __ offset();
+// OK to trash LR, because exception blob will kill it
+__ jump(OptoRuntime::exception_blob()->entry_point(), relocInfo::runtime_call_type, LR_tmp);
+assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
+__ end_a_stub();
+return offset;
+}
+int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf) {
+// Can't use any of the current frame's registers as we may have deopted
+// at a poll and everything can be live.
+MacroAssembler _masm(&cbuf);
+address base = __ start_a_stub(size_deopt_handler());
+if (base == NULL) {
+ciEnv::current()->record_failure("CodeCache is full");
+return 0;  // CodeBuffer::expand failed
+}
+int offset = __ offset();
+address deopt_pc = __ pc();
+#ifdef AARCH64
+// See LR saved by caller in sharedRuntime_arm.cpp
+// see also hse1 ws
+// see also LIR_Assembler::emit_deopt_handler
+__ raw_push(LR, LR); // preserve LR in both slots
+__ mov_relative_address(LR, deopt_pc);
+__ str(LR, Address(SP, 1 * wordSize)); // save deopt PC
+// OK to kill LR, because deopt blob will restore it from SP[0]
+__ jump(SharedRuntime::deopt_blob()->unpack(), relocInfo::runtime_call_type, LR_tmp);
+#else
+__ sub(SP, SP, wordSize); // make room for saved PC
+__ push(LR); // save LR that may be live when we get here
+__ mov_relative_address(LR, deopt_pc);
+__ str(LR, Address(SP, wordSize)); // save deopt PC
+__ pop(LR); // restore LR
+__ jump(SharedRuntime::deopt_blob()->unpack(), relocInfo::runtime_call_type, noreg);
+#endif
+assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
+__ end_a_stub();
+return offset;
+}
+const bool Matcher::match_rule_supported(int opcode) {
+if (!has_match_rule(opcode))
+return false;
+switch (opcode) {
+case Op_PopCountI:
+case Op_PopCountL:
+if (!UsePopCountInstruction)
+return false;
+break;
+case Op_LShiftCntV:
+case Op_RShiftCntV:
+case Op_AddVB:
+case Op_AddVS:
+case Op_AddVI:
+case Op_AddVL:
+case Op_SubVB:
+case Op_SubVS:
+case Op_SubVI:
+case Op_SubVL:
+case Op_MulVS:
+case Op_MulVI:
+case Op_LShiftVB:
+case Op_LShiftVS:
+case Op_LShiftVI:
+case Op_LShiftVL:
+case Op_RShiftVB:
+case Op_RShiftVS:
+case Op_RShiftVI:
+case Op_RShiftVL:
+case Op_URShiftVB:
+case Op_URShiftVS:
+case Op_URShiftVI:
+case Op_URShiftVL:
+case Op_AndV:
+case Op_OrV:
+case Op_XorV:
+return VM_Version::has_simd();
+case Op_LoadVector:
+case Op_StoreVector:
+case Op_AddVF:
+case Op_SubVF:
+case Op_MulVF:
+#ifdef AARCH64
+return VM_Version::has_simd();
+#else
+return VM_Version::has_vfp() || VM_Version::has_simd();
+#endif
+case Op_AddVD:
+case Op_SubVD:
+case Op_MulVD:
+case Op_DivVF:
+case Op_DivVD:
+#ifdef AARCH64
+return VM_Version::has_simd();
+#else
+return VM_Version::has_vfp();
+#endif
+}
+return true;  // Per default match rules are supported.
+}
+const bool Matcher::match_rule_supported_vector(int opcode, int vlen) {
+// TODO
+// identify extra cases that we might want to provide match rules for
+// e.g. Op_ vector nodes and other intrinsics while guarding with vlen
+bool ret_value = match_rule_supported(opcode);
+// Add rules here.
+return ret_value;  // Per default match rules are supported.
+}
+const bool Matcher::has_predicated_vectors(void) {
+return false;
+}
+const int Matcher::float_pressure(int default_pressure_threshold) {
+return default_pressure_threshold;
+}
+int Matcher::regnum_to_fpu_offset(int regnum) {
+return regnum - 32; // The FP registers are in the second chunk
+}
+// Vector width in bytes
+const int Matcher::vector_width_in_bytes(BasicType bt) {
+return MaxVectorSize;
+}
+// Vector ideal reg corresponding to specified size in bytes
+const int Matcher::vector_ideal_reg(int size) {
+assert(MaxVectorSize >= size, "");
+switch(size) {
+case  8: return Op_VecD;
+case 16: return Op_VecX;
+}
+ShouldNotReachHere();
+return 0;
+}
+const int Matcher::vector_shift_count_ideal_reg(int size) {
+return vector_ideal_reg(size);
+}
+// Limits on vector size (number of elements) loaded into vector.
+const int Matcher::max_vector_size(const BasicType bt) {
+assert(is_java_primitive(bt), "only primitive type vectors");
+return vector_width_in_bytes(bt)/type2aelembytes(bt);
+}
+const int Matcher::min_vector_size(const BasicType bt) {
+assert(is_java_primitive(bt), "only primitive type vectors");
+return 8/type2aelembytes(bt);
+}
+// ARM doesn't support misaligned vectors store/load.
+const bool Matcher::misaligned_vectors_ok() {
+return false;
+}
+// ARM doesn't support AES intrinsics
+const bool Matcher::pass_original_key_for_aes() {
+return false;
+}
+const bool Matcher::convL2FSupported(void) {
+#ifdef AARCH64
+return true;
+#else
+return false;
+#endif
+}
+// Is this branch offset short enough that a short branch can be used?
+//
+// NOTE: If the platform does not provide any short branch variants, then
+//       this method should return false for offset 0.
+bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
+// The passed offset is relative to address of the branch.
+// On ARM a branch displacement is calculated relative to address
+// of the branch + 8.
+//
+// offset -= 8;
+// return (Assembler::is_simm24(offset));
+return false;
+}
+const bool Matcher::isSimpleConstant64(jlong value) {
+// Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
+#ifdef AARCH64
+return (value == 0);
+#else
+return false;
+#endif
+}
+// No scaling for the parameter the ClearArray node.
+const bool Matcher::init_array_count_is_in_bytes = true;
+#ifdef AARCH64
+const int Matcher::long_cmove_cost() { return 1; }
+#else
+// Needs 2 CMOV's for longs.
+const int Matcher::long_cmove_cost() { return 2; }
+#endif
+#ifdef AARCH64
+const int Matcher::float_cmove_cost() { return 1; }
+#else
+// CMOVF/CMOVD are expensive on ARM.
+const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
+#endif
+// Does the CPU require late expand (see block.cpp for description of late expand)?
+const bool Matcher::require_postalloc_expand = false;
+// Do we need to mask the count passed to shift instructions or does
+// the cpu only look at the lower 5/6 bits anyway?
+// FIXME: does this handle vector shifts as well?
+#ifdef AARCH64
+const bool Matcher::need_masked_shift_count = false;
+#else
+const bool Matcher::need_masked_shift_count = true;
+#endif
+const bool Matcher::convi2l_type_required = true;
+// Should the Matcher clone shifts on addressing modes, expecting them
+// to be subsumed into complex addressing expressions or compute them
+// into registers?
+bool Matcher::clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
+return clone_base_plus_offset_address(m, mstack, address_visited);
+}
+void Compile::reshape_address(AddPNode* addp) {
+}
+bool Matcher::narrow_oop_use_complex_address() {
+NOT_LP64(ShouldNotCallThis());
+assert(UseCompressedOops, "only for compressed oops code");
+return false;
+}
+bool Matcher::narrow_klass_use_complex_address() {
+NOT_LP64(ShouldNotCallThis());
+assert(UseCompressedClassPointers, "only for compressed klass code");
+return false;
+}
+bool Matcher::const_oop_prefer_decode() {
+NOT_LP64(ShouldNotCallThis());
+return true;
+}
+bool Matcher::const_klass_prefer_decode() {
+NOT_LP64(ShouldNotCallThis());
+return true;
+}
+// Is it better to copy float constants, or load them directly from memory?
+// Intel can load a float constant from a direct address, requiring no
+// extra registers.  Most RISCs will have to materialize an address into a
+// register first, so they would do better to copy the constant from stack.
+const bool Matcher::rematerialize_float_constants = false;
+// If CPU can load and store mis-aligned doubles directly then no fixup is
+// needed.  Else we split the double into 2 integer pieces and move it
+// piece-by-piece.  Only happens when passing doubles into C code as the
+// Java calling convention forces doubles to be aligned.
+#ifdef AARCH64
+// On stack replacement support:
+// We don't need Load[DL]_unaligned support, because interpreter stack
+// has correct alignment
+const bool Matcher::misaligned_doubles_ok = true;
+#else
+const bool Matcher::misaligned_doubles_ok = false;
+#endif
+// No-op on ARM.
+void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {
+}
+// Advertise here if the CPU requires explicit rounding operations
+// to implement the UseStrictFP mode.
+const bool Matcher::strict_fp_requires_explicit_rounding = false;
+// Are floats converted to double when stored to stack during deoptimization?
+// ARM does not handle callee-save floats.
+bool Matcher::float_in_double() {
+return false;
+}
+// Do ints take an entire long register or just half?
+// Note that we if-def off of _LP64.
+// The relevant question is how the int is callee-saved.  In _LP64
+// the whole long is written but de-opt'ing will have to extract
+// the relevant 32 bits, in not-_LP64 only the low 32 bits is written.
+#ifdef _LP64
+const bool Matcher::int_in_long = true;
+#else
+const bool Matcher::int_in_long = false;
+#endif
+// Return whether or not this register is ever used as an argument.  This
+// function is used on startup to build the trampoline stubs in generateOptoStub.
+// Registers not mentioned will be killed by the VM call in the trampoline, and
+// arguments in those registers not be available to the callee.
+bool Matcher::can_be_java_arg( int reg ) {
+#ifdef AARCH64
+if (reg >= R_R0_num && reg < R_R8_num) return true;
+if (reg >= R_V0_num && reg <= R_V7b_num && ((reg & 3) < 2)) return true;
+#else
+if (reg == R_R0_num ||
+reg == R_R1_num ||
+reg == R_R2_num ||
+reg == R_R3_num) return true;
+if (reg >= R_S0_num &&
+reg <= R_S13_num) return true;
+#endif
+return false;
+}
+bool Matcher::is_spillable_arg( int reg ) {
+return can_be_java_arg(reg);
+}
+bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
+return false;
+}
+// Register for DIVI projection of divmodI
+RegMask Matcher::divI_proj_mask() {
+ShouldNotReachHere();
+return RegMask();
+}
+// Register for MODI projection of divmodI
+RegMask Matcher::modI_proj_mask() {
+ShouldNotReachHere();
+return RegMask();
+}
+// Register for DIVL projection of divmodL
+RegMask Matcher::divL_proj_mask() {
+ShouldNotReachHere();
+return RegMask();
+}
+// Register for MODL projection of divmodL
+RegMask Matcher::modL_proj_mask() {
+ShouldNotReachHere();
+return RegMask();
+}
+const RegMask Matcher::method_handle_invoke_SP_save_mask() {
+return FP_REGP_mask();
+}
+bool maybe_far_call(const CallNode *n) {
+return !MacroAssembler::_reachable_from_cache(n->as_Call()->entry_point());
+}
+bool maybe_far_call(const MachCallNode *n) {
+return !MacroAssembler::_reachable_from_cache(n->as_MachCall()->entry_point());
+}
+%}
+//----------ENCODING BLOCK-----------------------------------------------------
+// This block specifies the encoding classes used by the compiler to output
+// byte streams.  Encoding classes are parameterized macros used by
+// Machine Instruction Nodes in order to generate the bit encoding of the
+// instruction.  Operands specify their base encoding interface with the
+// interface keyword.  There are currently supported four interfaces,
+// REG_INTER, CONST_INTER, MEMORY_INTER, & COND_INTER.  REG_INTER causes an
+// operand to generate a function which returns its register number when
+// queried.   CONST_INTER causes an operand to generate a function which
+// returns the value of the constant when queried.  MEMORY_INTER causes an
+// operand to generate four functions which return the Base Register, the
+// Index Register, the Scale Value, and the Offset Value of the operand when
+// queried.  COND_INTER causes an operand to generate six functions which
+// return the encoding code (ie - encoding bits for the instruction)
+// associated with each basic boolean condition for a conditional instruction.
+//
+// Instructions specify two basic values for encoding.  Again, a function
+// is available to check if the constant displacement is an oop. They use the
+// ins_encode keyword to specify their encoding classes (which must be
+// a sequence of enc_class names, and their parameters, specified in
+// the encoding block), and they use the
+// opcode keyword to specify, in order, their primary, secondary, and
+// tertiary opcode.  Only the opcode sections which a particular instruction
+// needs for encoding need to be specified.
+encode %{
+enc_class call_epilog %{
+// nothing
+%}
+enc_class Java_To_Runtime (method meth) %{
+// CALL directly to the runtime
+emit_call_reloc(cbuf, as_MachCall(), $meth, runtime_call_Relocation::spec());
+%}
+enc_class Java_Static_Call (method meth) %{
+// CALL to fixup routine.  Fixup routine uses ScopeDesc info to determine
+// who we intended to call.
+if ( !_method) {
+emit_call_reloc(cbuf, as_MachCall(), $meth, runtime_call_Relocation::spec());
+} else {
+int method_index = resolved_method_index(cbuf);
+RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
+: static_call_Relocation::spec(method_index);
+emit_call_reloc(cbuf, as_MachCall(), $meth, rspec);
+// Emit stubs for static call.
+address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
+if (stub == NULL) {
+ciEnv::current()->record_failure("CodeCache is full");
+return;
+}
+}
+%}
+enc_class save_last_PC %{
+// preserve mark
+address mark = cbuf.insts()->mark();
+debug_only(int off0 = cbuf.insts_size());
+MacroAssembler _masm(&cbuf);
+int ret_addr_offset = as_MachCall()->ret_addr_offset();
+__ adr(LR, mark + ret_addr_offset);
+__ str(LR, Address(Rthread, JavaThread::last_Java_pc_offset()));
+debug_only(int off1 = cbuf.insts_size());
+assert(off1 - off0 == 2 * Assembler::InstructionSize, "correct size prediction");
+// restore mark
+cbuf.insts()->set_mark(mark);
+%}
+enc_class preserve_SP %{
+// preserve mark
+address mark = cbuf.insts()->mark();
+debug_only(int off0 = cbuf.insts_size());
+MacroAssembler _masm(&cbuf);
+// FP is preserved across all calls, even compiled calls.
+// Use it to preserve SP in places where the callee might change the SP.
+__ mov(Rmh_SP_save, SP);
+debug_only(int off1 = cbuf.insts_size());
+assert(off1 - off0 == 4, "correct size prediction");
+// restore mark
+cbuf.insts()->set_mark(mark);
+%}
+enc_class restore_SP %{
+MacroAssembler _masm(&cbuf);
+__ mov(SP, Rmh_SP_save);
+%}
+enc_class Java_Dynamic_Call (method meth) %{
+MacroAssembler _masm(&cbuf);
+Register R8_ic_reg = reg_to_register_object(Matcher::inline_cache_reg_encode());
+assert(R8_ic_reg == Ricklass, "should be");
+__ set_inst_mark();
+#ifdef AARCH64
+// TODO: see C1 LIR_Assembler::ic_call()
+InlinedAddress oop_literal((address)Universe::non_oop_word());
+int offset = __ offset();
+int fixed_size = mov_oop_size * 4;
+if (VM_Version::prefer_moves_over_load_literal()) {
+uintptr_t val = (uintptr_t)Universe::non_oop_word();
+__ movz(R8_ic_reg, (val >>  0) & 0xffff,  0);
+__ movk(R8_ic_reg, (val >> 16) & 0xffff, 16);
+__ movk(R8_ic_reg, (val >> 32) & 0xffff, 32);
+__ movk(R8_ic_reg, (val >> 48) & 0xffff, 48);
+} else {
+__ ldr_literal(R8_ic_reg, oop_literal);
+}
+assert(__ offset() - offset == fixed_size, "bad mov_oop size");
+#else
+__ movw(R8_ic_reg, ((unsigned int)Universe::non_oop_word()) & 0xffff);
+__ movt(R8_ic_reg, ((unsigned int)Universe::non_oop_word()) >> 16);
+#endif
+address  virtual_call_oop_addr = __ inst_mark();
+// CALL to fixup routine.  Fixup routine uses ScopeDesc info to determine
+// who we intended to call.
+int method_index = resolved_method_index(cbuf);
+__ relocate(virtual_call_Relocation::spec(virtual_call_oop_addr, method_index));
+emit_call_reloc(cbuf, as_MachCall(), $meth, RelocationHolder::none);
+#ifdef AARCH64
+if (!VM_Version::prefer_moves_over_load_literal()) {
+Label skip_literal;
+__ b(skip_literal);
+int off2 = __ offset();
+__ bind_literal(oop_literal);
+if (__ offset() - off2 == wordSize) {
+// no padding, so insert nop for worst-case sizing
+__ nop();
+}
+__ bind(skip_literal);
+}
+#endif
+%}
+enc_class LdReplImmI(immI src, regD dst, iRegI tmp, int cnt, int wth) %{
+// FIXME: load from constant table?
+// Load a constant replicated "count" times with width "width"
+int count = $cnt$$constant;
+int width = $wth$$constant;
+assert(count*width == 4, "sanity");
+int val = $src$$constant;
+if (width < 4) {
+int bit_width = width * 8;
+val &= (((int)1) << bit_width) - 1; // mask off sign bits
+for (int i = 0; i < count - 1; i++) {
+val |= (val << bit_width);
+}
+}
+MacroAssembler _masm(&cbuf);
+if (val == -1) {
+__ mvn($tmp$$Register, 0);
+} else if (val == 0) {
+__ mov($tmp$$Register, 0);
+} else {
+__ movw($tmp$$Register, val & 0xffff);
+__ movt($tmp$$Register, (unsigned int)val >> 16);
+}
+__ fmdrr($dst$$FloatRegister, $tmp$$Register, $tmp$$Register);
+%}
+enc_class LdReplImmF(immF src, regD dst, iRegI tmp) %{
+// Replicate float con 2 times and pack into vector (8 bytes) in regD.
+float fval = $src$$constant;
+int val = *((int*)&fval);
+MacroAssembler _masm(&cbuf);
+if (val == -1) {
+__ mvn($tmp$$Register, 0);
+} else if (val == 0) {
+__ mov($tmp$$Register, 0);
+} else {
+__ movw($tmp$$Register, val & 0xffff);
+__ movt($tmp$$Register, (unsigned int)val >> 16);
+}
+__ fmdrr($dst$$FloatRegister, $tmp$$Register, $tmp$$Register);
+%}
+enc_class enc_String_Compare(R0RegP str1, R1RegP str2, R2RegI cnt1, R3RegI cnt2, iRegI result, iRegI tmp1, iRegI tmp2) %{
+Label Ldone, Lloop;
+MacroAssembler _masm(&cbuf);
+Register   str1_reg = $str1$$Register;
+Register   str2_reg = $str2$$Register;
+Register   cnt1_reg = $cnt1$$Register; // int
+Register   cnt2_reg = $cnt2$$Register; // int
+Register   tmp1_reg = $tmp1$$Register;
+Register   tmp2_reg = $tmp2$$Register;
+Register result_reg = $result$$Register;
+assert_different_registers(str1_reg, str2_reg, cnt1_reg, cnt2_reg, tmp1_reg, tmp2_reg);
+// Compute the minimum of the string lengths(str1_reg) and the
+// difference of the string lengths (stack)
+// See if the lengths are different, and calculate min in str1_reg.
+// Stash diff in tmp2 in case we need it for a tie-breaker.
+__ subs_32(tmp2_reg, cnt1_reg, cnt2_reg);
+#ifdef AARCH64
+Label Lskip;
+__ _lsl_w(cnt1_reg, cnt1_reg, exact_log2(sizeof(jchar))); // scale the limit
+__ b(Lskip, mi);
+__ _lsl_w(cnt1_reg, cnt2_reg, exact_log2(sizeof(jchar))); // scale the limit
+__ bind(Lskip);
+#else
+__ mov(cnt1_reg, AsmOperand(cnt1_reg, lsl, exact_log2(sizeof(jchar)))); // scale the limit
+__ mov(cnt1_reg, AsmOperand(cnt2_reg, lsl, exact_log2(sizeof(jchar))), pl); // scale the limit
+#endif
+// reallocate cnt1_reg, cnt2_reg, result_reg
+// Note:  limit_reg holds the string length pre-scaled by 2
+Register limit_reg = cnt1_reg;
+Register  chr2_reg = cnt2_reg;
+Register  chr1_reg = tmp1_reg;
+// str{12} are the base pointers
+// Is the minimum length zero?
+__ cmp_32(limit_reg, 0);
+if (result_reg != tmp2_reg) {
+__ mov(result_reg, tmp2_reg, eq);
+}
+__ b(Ldone, eq);
+// Load first characters
+__ ldrh(chr1_reg, Address(str1_reg, 0));
+__ ldrh(chr2_reg, Address(str2_reg, 0));
+// Compare first characters
+__ subs(chr1_reg, chr1_reg, chr2_reg);
+if (result_reg != chr1_reg) {
+__ mov(result_reg, chr1_reg, ne);
+}
+__ b(Ldone, ne);
+{
+// Check after comparing first character to see if strings are equivalent
+// Check if the strings start at same location
+__ cmp(str1_reg, str2_reg);
+// Check if the length difference is zero
+__ cond_cmp(tmp2_reg, 0, eq);
+__ mov(result_reg, 0, eq); // result is zero
+__ b(Ldone, eq);
+// Strings might not be equal
+}
+__ subs(chr1_reg, limit_reg, 1 * sizeof(jchar));
+if (result_reg != tmp2_reg) {
+__ mov(result_reg, tmp2_reg, eq);
+}
+__ b(Ldone, eq);
+// Shift str1_reg and str2_reg to the end of the arrays, negate limit
+__ add(str1_reg, str1_reg, limit_reg);
+__ add(str2_reg, str2_reg, limit_reg);
+__ neg(limit_reg, chr1_reg);  // limit = -(limit-2)
+// Compare the rest of the characters
+__ bind(Lloop);
+__ ldrh(chr1_reg, Address(str1_reg, limit_reg));
+__ ldrh(chr2_reg, Address(str2_reg, limit_reg));
+__ subs(chr1_reg, chr1_reg, chr2_reg);
+if (result_reg != chr1_reg) {
+__ mov(result_reg, chr1_reg, ne);
+}
+__ b(Ldone, ne);
+__ adds(limit_reg, limit_reg, sizeof(jchar));
+__ b(Lloop, ne);
+// If strings are equal up to min length, return the length difference.
+if (result_reg != tmp2_reg) {
+__ mov(result_reg, tmp2_reg);
+}
+// Otherwise, return the difference between the first mismatched chars.
+__ bind(Ldone);
+%}
+enc_class enc_String_Equals(R0RegP str1, R1RegP str2, R2RegI cnt, iRegI result, iRegI tmp1, iRegI tmp2) %{
+Label Lword_loop, Lpost_word, Lchar, Lchar_loop, Ldone, Lequal;
+MacroAssembler _masm(&cbuf);
+Register   str1_reg = $str1$$Register;
+Register   str2_reg = $str2$$Register;
+Register    cnt_reg = $cnt$$Register; // int
+Register   tmp1_reg = $tmp1$$Register;
+Register   tmp2_reg = $tmp2$$Register;
+Register result_reg = $result$$Register;
+assert_different_registers(str1_reg, str2_reg, cnt_reg, tmp1_reg, tmp2_reg, result_reg);
+__ cmp(str1_reg, str2_reg); //same char[] ?
+__ b(Lequal, eq);
+__ cbz_32(cnt_reg, Lequal); // count == 0
+//rename registers
+Register limit_reg = cnt_reg;
+Register  chr1_reg = tmp1_reg;
+Register  chr2_reg = tmp2_reg;
+__ logical_shift_left(limit_reg, limit_reg, exact_log2(sizeof(jchar)));
+//check for alignment and position the pointers to the ends
+__ orr(chr1_reg, str1_reg, str2_reg);
+__ tst(chr1_reg, 0x3);
+// notZero means at least one not 4-byte aligned.
+// We could optimize the case when both arrays are not aligned
+// but it is not frequent case and it requires additional checks.
+__ b(Lchar, ne);
+// Compare char[] arrays aligned to 4 bytes.
+__ char_arrays_equals(str1_reg, str2_reg, limit_reg, result_reg,
+chr1_reg, chr2_reg, Ldone);
+__ b(Lequal); // equal
+// char by char compare
+__ bind(Lchar);
+__ mov(result_reg, 0);
+__ add(str1_reg, limit_reg, str1_reg);
+__ add(str2_reg, limit_reg, str2_reg);
+__ neg(limit_reg, limit_reg); //negate count
+// Lchar_loop
+__ bind(Lchar_loop);
+__ ldrh(chr1_reg, Address(str1_reg, limit_reg));
+__ ldrh(chr2_reg, Address(str2_reg, limit_reg));
+__ cmp(chr1_reg, chr2_reg);
+__ b(Ldone, ne);
+__ adds(limit_reg, limit_reg, sizeof(jchar));
+__ b(Lchar_loop, ne);
+__ bind(Lequal);
+__ mov(result_reg, 1);  //equal
+__ bind(Ldone);
+%}
+enc_class enc_Array_Equals(R0RegP ary1, R1RegP ary2, iRegI tmp1, iRegI tmp2, iRegI tmp3, iRegI result) %{
+Label Lvector, Ldone, Lloop, Lequal;
+MacroAssembler _masm(&cbuf);
+Register   ary1_reg = $ary1$$Register;
+Register   ary2_reg = $ary2$$Register;
+Register   tmp1_reg = $tmp1$$Register;
+Register   tmp2_reg = $tmp2$$Register;
+Register   tmp3_reg = $tmp3$$Register;
+Register result_reg = $result$$Register;
+assert_different_registers(ary1_reg, ary2_reg, tmp1_reg, tmp2_reg, tmp3_reg, result_reg);
+int length_offset  = arrayOopDesc::length_offset_in_bytes();
+int base_offset    = arrayOopDesc::base_offset_in_bytes(T_CHAR);
+// return true if the same array
+#ifdef AARCH64
+__ cmp(ary1_reg, ary2_reg);
+__ b(Lequal, eq);
+__ mov(result_reg, 0);
+__ cbz(ary1_reg, Ldone); // not equal
+__ cbz(ary2_reg, Ldone); // not equal
+#else
+__ teq(ary1_reg, ary2_reg);
+__ mov(result_reg, 1, eq);
+__ b(Ldone, eq); // equal
+__ tst(ary1_reg, ary1_reg);
+__ mov(result_reg, 0, eq);
+__ b(Ldone, eq);    // not equal
+__ tst(ary2_reg, ary2_reg);
+__ mov(result_reg, 0, eq);
+__ b(Ldone, eq);    // not equal
+#endif
+//load the lengths of arrays
+__ ldr_s32(tmp1_reg, Address(ary1_reg, length_offset)); // int
+__ ldr_s32(tmp2_reg, Address(ary2_reg, length_offset)); // int
+// return false if the two arrays are not equal length
+#ifdef AARCH64
+__ cmp_w(tmp1_reg, tmp2_reg);
+__ b(Ldone, ne);    // not equal
+__ cbz_w(tmp1_reg, Lequal); // zero-length arrays are equal
+#else
+__ teq_32(tmp1_reg, tmp2_reg);
+__ mov(result_reg, 0, ne);
+__ b(Ldone, ne);    // not equal
+__ tst(tmp1_reg, tmp1_reg);
+__ mov(result_reg, 1, eq);
+__ b(Ldone, eq);    // zero-length arrays are equal
+#endif
+// load array addresses
+__ add(ary1_reg, ary1_reg, base_offset);
+__ add(ary2_reg, ary2_reg, base_offset);
+// renaming registers
+Register chr1_reg  =  tmp3_reg;   // for characters in ary1
+Register chr2_reg  =  tmp2_reg;   // for characters in ary2
+Register limit_reg =  tmp1_reg;   // length
+// set byte count
+__ logical_shift_left_32(limit_reg, limit_reg, exact_log2(sizeof(jchar)));
+// Compare char[] arrays aligned to 4 bytes.
+__ char_arrays_equals(ary1_reg, ary2_reg, limit_reg, result_reg,
+chr1_reg, chr2_reg, Ldone);
+__ bind(Lequal);
+__ mov(result_reg, 1);  //equal
+__ bind(Ldone);
+%}
+%}
+//----------FRAME--------------------------------------------------------------
+// Definition of frame structure and management information.
+//
+//  S T A C K   L A Y O U T    Allocators stack-slot number
+//                             |   (to get allocators register number
+//  G  Owned by    |        |  v    add VMRegImpl::stack0)
+//  r   CALLER     |        |
+//  o     |        +--------+      pad to even-align allocators stack-slot
+//  w     V        |  pad0  |        numbers; owned by CALLER
+//  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
+//  h     ^        |   in   |  5
+//        |        |  args  |  4   Holes in incoming args owned by SELF
+//  |     |        |        |  3
+//  |     |        +--------+
+//  V     |        | old out|      Empty on Intel, window on Sparc
+//        |    old |preserve|      Must be even aligned.
+//        |     SP-+--------+----> Matcher::_old_SP, 8 (or 16 in LP64)-byte aligned
+//        |        |   in   |  3   area for Intel ret address
+//     Owned by    |preserve|      Empty on Sparc.
+//       SELF      +--------+
+//        |        |  pad2  |  2   pad to align old SP
+//        |        +--------+  1
+//        |        | locks  |  0
+//        |        +--------+----> VMRegImpl::stack0, 8 (or 16 in LP64)-byte aligned
+//        |        |  pad1  | 11   pad to align new SP
+//        |        +--------+
+//        |        |        | 10
+//        |        | spills |  9   spills
+//        V        |        |  8   (pad0 slot for callee)
+//      -----------+--------+----> Matcher::_out_arg_limit, unaligned
+//        ^        |  out   |  7
+//        |        |  args  |  6   Holes in outgoing args owned by CALLEE
+//     Owned by    +--------+
+//      CALLEE     | new out|  6   Empty on Intel, window on Sparc
+//        |    new |preserve|      Must be even-aligned.
+//        |     SP-+--------+----> Matcher::_new_SP, even aligned
+//        |        |        |
+//
+// Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
+//         known from SELF's arguments and the Java calling convention.
+//         Region 6-7 is determined per call site.
+// Note 2: If the calling convention leaves holes in the incoming argument
+//         area, those holes are owned by SELF.  Holes in the outgoing area
+//         are owned by the CALLEE.  Holes should not be nessecary in the
+//         incoming area, as the Java calling convention is completely under
+//         the control of the AD file.  Doubles can be sorted and packed to
+//         avoid holes.  Holes in the outgoing arguments may be nessecary for
+//         varargs C calling conventions.
+// Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
+//         even aligned with pad0 as needed.
+//         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
+//         region 6-11 is even aligned; it may be padded out more so that
+//         the region from SP to FP meets the minimum stack alignment.
+frame %{
+// What direction does stack grow in (assumed to be same for native & Java)
+stack_direction(TOWARDS_LOW);
+// These two registers define part of the calling convention
+// between compiled code and the interpreter.
+inline_cache_reg(R_Ricklass);          // Inline Cache Register or Method* for I2C
+interpreter_method_oop_reg(R_Rmethod); // Method Oop Register when calling interpreter
+// Optional: name the operand used by cisc-spilling to access [stack_pointer + offset]
+cisc_spilling_operand_name(indOffset);
+// Number of stack slots consumed by a Monitor enter
+sync_stack_slots(1 * VMRegImpl::slots_per_word);
+// Compiled code's Frame Pointer
+#ifdef AARCH64
+frame_pointer(R_SP);
+#else
+frame_pointer(R_R13);
+#endif
+// Stack alignment requirement
+stack_alignment(StackAlignmentInBytes);
+//  LP64: Alignment size in bytes (128-bit -> 16 bytes)
+// !LP64: Alignment size in bytes (64-bit  ->  8 bytes)
+// Number of stack slots between incoming argument block and the start of
+// a new frame.  The PROLOG must add this many slots to the stack.  The
+// EPILOG must remove this many slots.
+// FP + LR
+in_preserve_stack_slots(2 * VMRegImpl::slots_per_word);
+// Number of outgoing stack slots killed above the out_preserve_stack_slots
+// for calls to C.  Supports the var-args backing area for register parms.
+// ADLC doesn't support parsing expressions, so I folded the math by hand.
+varargs_C_out_slots_killed( 0);
+// The after-PROLOG location of the return address.  Location of
+// return address specifies a type (REG or STACK) and a number
+// representing the register number (i.e. - use a register name) or
+// stack slot.
+// Ret Addr is on stack in slot 0 if no locks or verification or alignment.
+// Otherwise, it is above the locks and verification slot and alignment word
+return_addr(STACK - 1*VMRegImpl::slots_per_word +
+round_to((Compile::current()->in_preserve_stack_slots() +
+Compile::current()->fixed_slots()),
+stack_alignment_in_slots()));
+// Body of function which returns an OptoRegs array locating
+// arguments either in registers or in stack slots for calling
+// java
+calling_convention %{
+(void) SharedRuntime::java_calling_convention(sig_bt, regs, length, is_outgoing);
+%}
+// Body of function which returns an OptoRegs array locating
+// arguments either in registers or in stack slots for callin
+// C.
+c_calling_convention %{
+// This is obviously always outgoing
+(void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
+%}
+// Location of compiled Java return values.  Same as C
+return_value %{
+return c2::return_value(ideal_reg);
+%}
+%}
+//----------ATTRIBUTES---------------------------------------------------------
+//----------Instruction Attributes---------------------------------------------
+ins_attrib ins_cost(DEFAULT_COST); // Required cost attribute
+ins_attrib ins_size(32);           // Required size attribute (in bits)
+ins_attrib ins_short_branch(0);    // Required flag: is this instruction a
+// non-matching short branch variant of some
+// long branch?
+//----------OPERANDS-----------------------------------------------------------
+// Operand definitions must precede instruction definitions for correct parsing
+// in the ADLC because operands constitute user defined types which are used in
+// instruction definitions.
+//----------Simple Operands----------------------------------------------------
+// Immediate Operands
+// Integer Immediate: 32-bit
+operand immI() %{
+match(ConI);
+op_cost(0);
+// formats are generated automatically for constants and base registers
+format %{ %}
+interface(CONST_INTER);
+%}
+// Integer Immediate: 8-bit unsigned - for VMOV
+operand immU8() %{
+predicate(0 <= n->get_int() && (n->get_int() <= 255));
+match(ConI);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+// Integer Immediate: 16-bit
+operand immI16() %{
+predicate((n->get_int() >> 16) == 0 && VM_Version::supports_movw());
+match(ConI);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+#ifndef AARCH64
+// Integer Immediate: offset for half and double word loads and stores
+operand immIHD() %{
+predicate(is_memoryHD(n->get_int()));
+match(ConI);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+// Integer Immediate: offset for fp loads and stores
+operand immIFP() %{
+predicate(is_memoryfp(n->get_int()) && ((n->get_int() & 3) == 0));
+match(ConI);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+#endif
+// Valid scale values for addressing modes and shifts
+operand immU5() %{
+predicate(0 <= n->get_int() && (n->get_int() <= 31));
+match(ConI);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+// Integer Immediate: 6-bit
+operand immU6Big() %{
+predicate(n->get_int() >= 32 && n->get_int() <= 63);
+match(ConI);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+// Integer Immediate: 0-bit
+operand immI0() %{
+predicate(n->get_int() == 0);
+match(ConI);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+// Integer Immediate: the value 1
+operand immI_1() %{
+predicate(n->get_int() == 1);
+match(ConI);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+// Integer Immediate: the value 2
+operand immI_2() %{
+predicate(n->get_int() == 2);
+match(ConI);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+// Integer Immediate: the value 3
+operand immI_3() %{
+predicate(n->get_int() == 3);
+match(ConI);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+// Integer Immediate: the value 4
+operand immI_4() %{
+predicate(n->get_int() == 4);
+match(ConI);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+// Integer Immediate: the value 8
+operand immI_8() %{
+predicate(n->get_int() == 8);
+match(ConI);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+// Int Immediate non-negative
+operand immU31()
+%{
+predicate(n->get_int() >= 0);
+match(ConI);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+// Integer Immediate: the values 32-63
+operand immI_32_63() %{
+predicate(n->get_int() >= 32 && n->get_int() <= 63);
+match(ConI);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+// Immediates for special shifts (sign extend)
+// Integer Immediate: the value 16
+operand immI_16() %{
+predicate(n->get_int() == 16);
+match(ConI);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+// Integer Immediate: the value 24
+operand immI_24() %{
+predicate(n->get_int() == 24);
+match(ConI);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+// Integer Immediate: the value 255
+operand immI_255() %{
+predicate( n->get_int() == 255 );
+match(ConI);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+// Integer Immediate: the value 65535
+operand immI_65535() %{
+predicate(n->get_int() == 65535);
+match(ConI);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+// Integer Immediates for arithmetic instructions
+operand aimmI() %{
+predicate(is_aimm(n->get_int()));
+match(ConI);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+operand aimmIneg() %{
+predicate(is_aimm(-n->get_int()));
+match(ConI);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+operand aimmU31() %{
+predicate((0 <= n->get_int()) && is_aimm(n->get_int()));
+match(ConI);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+// Integer Immediates for logical instructions
+operand limmI() %{
+predicate(is_limmI(n->get_int()));
+match(ConI);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+operand limmIlow8() %{
+predicate(is_limmI_low(n->get_int(), 8));
+match(ConI);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+operand limmU31() %{
+predicate(0 <= n->get_int() && is_limmI(n->get_int()));
+match(ConI);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+operand limmIn() %{
+predicate(is_limmI(~n->get_int()));
+match(ConI);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+#ifdef AARCH64
+// Long Immediate: for logical instruction
+operand limmL() %{
+predicate(is_limmL(n->get_long()));
+match(ConL);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+operand limmLn() %{
+predicate(is_limmL(~n->get_long()));
+match(ConL);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+// Long Immediate: for arithmetic instruction
+operand aimmL() %{
+predicate(is_aimm(n->get_long()));
+match(ConL);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+operand aimmLneg() %{
+predicate(is_aimm(-n->get_long()));
+match(ConL);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+#endif // AARCH64
+// Long Immediate: the value FF
+operand immL_FF() %{
+predicate( n->get_long() == 0xFFL );
+match(ConL);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+// Long Immediate: the value FFFF
+operand immL_FFFF() %{
+predicate( n->get_long() == 0xFFFFL );
+match(ConL);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+// Pointer Immediate: 32 or 64-bit
+operand immP() %{
+match(ConP);
+op_cost(5);
+// formats are generated automatically for constants and base registers
+format %{ %}
+interface(CONST_INTER);
+%}
+operand immP0() %{
+predicate(n->get_ptr() == 0);
+match(ConP);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+operand immP_poll() %{
+predicate(n->get_ptr() != 0 && n->get_ptr() == (intptr_t)os::get_polling_page());
+match(ConP);
+// formats are generated automatically for constants and base registers
+format %{ %}
+interface(CONST_INTER);
+%}
+// Pointer Immediate
+operand immN()
+%{
+match(ConN);
+op_cost(10);
+format %{ %}
+interface(CONST_INTER);
+%}
+operand immNKlass()
+%{
+match(ConNKlass);
+op_cost(10);
+format %{ %}
+interface(CONST_INTER);
+%}
+// NULL Pointer Immediate
+operand immN0()
+%{
+predicate(n->get_narrowcon() == 0);
+match(ConN);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+operand immL() %{
+match(ConL);
+op_cost(40);
+// formats are generated automatically for constants and base registers
+format %{ %}
+interface(CONST_INTER);
+%}
+operand immL0() %{
+predicate(n->get_long() == 0L);
+match(ConL);
+op_cost(0);
+// formats are generated automatically for constants and base registers
+format %{ %}
+interface(CONST_INTER);
+%}
+// Long Immediate: 16-bit
+operand immL16() %{
+predicate(n->get_long() >= 0 && n->get_long() < (1<<16)  && VM_Version::supports_movw());
+match(ConL);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+// Long Immediate: low 32-bit mask
+operand immL_32bits() %{
+predicate(n->get_long() == 0xFFFFFFFFL);
+match(ConL);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+// Double Immediate
+operand immD() %{
+match(ConD);
+op_cost(40);
+format %{ %}
+interface(CONST_INTER);
+%}
+// Double Immediate: +0.0d.
+operand immD0() %{
+predicate(jlong_cast(n->getd()) == 0);
+match(ConD);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+operand imm8D() %{
+predicate(Assembler::double_num(n->getd()).can_be_imm8());
+match(ConD);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+// Float Immediate
+operand immF() %{
+match(ConF);
+op_cost(20);
+format %{ %}
+interface(CONST_INTER);
+%}
+// Float Immediate: +0.0f
+operand immF0() %{
+predicate(jint_cast(n->getf()) == 0);
+match(ConF);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+// Float Immediate: encoded as 8 bits
+operand imm8F() %{
+predicate(Assembler::float_num(n->getf()).can_be_imm8());
+match(ConF);
+op_cost(0);
+format %{ %}
+interface(CONST_INTER);
+%}
+// Integer Register Operands
+// Integer Register
+operand iRegI() %{
+constraint(ALLOC_IN_RC(int_reg));
+match(RegI);
+match(R0RegI);
+match(R1RegI);
+match(R2RegI);
+match(R3RegI);
+#ifdef AARCH64
+match(ZRRegI);
+#else
+match(R12RegI);
+#endif
+format %{ %}
+interface(REG_INTER);
+%}
+// Pointer Register
+operand iRegP() %{
+constraint(ALLOC_IN_RC(ptr_reg));
+match(RegP);
+match(R0RegP);
+match(R1RegP);
+match(R2RegP);
+match(RExceptionRegP);
+match(R8RegP);
+match(R9RegP);
+match(RthreadRegP); // FIXME: move to sp_ptr_RegP?
+match(R12RegP);
+match(LRRegP);
+match(sp_ptr_RegP);
+match(store_ptr_RegP);
+format %{ %}
+interface(REG_INTER);
+%}
+// GPRs + Rthread + SP
+operand sp_ptr_RegP() %{
+constraint(ALLOC_IN_RC(sp_ptr_reg));
+match(RegP);
+match(iRegP);
+match(SPRegP); // FIXME: check cost
+format %{ %}
+interface(REG_INTER);
+%}
+#ifdef AARCH64
+// Like sp_ptr_reg, but exclude regs (Aarch64 SP) that can't be
+// stored directly.  Includes ZR, so can't be used as a destination.
+operand store_ptr_RegP() %{
+constraint(ALLOC_IN_RC(store_ptr_reg));
+match(RegP);
+match(iRegP);
+match(ZRRegP);
+format %{ %}
+interface(REG_INTER);
+%}
+operand store_RegI() %{
+constraint(ALLOC_IN_RC(store_reg));
+match(RegI);
+match(iRegI);
+match(ZRRegI);
+format %{ %}
+interface(REG_INTER);
+%}
+operand store_RegL() %{
+constraint(ALLOC_IN_RC(store_ptr_reg));
+match(RegL);
+match(iRegL);
+match(ZRRegL);
+format %{ %}
+interface(REG_INTER);
+%}
+operand store_RegN() %{
+constraint(ALLOC_IN_RC(store_reg));
+match(RegN);
+match(iRegN);
+match(ZRRegN);
+format %{ %}
+interface(REG_INTER);
+%}
+#endif
+operand R0RegP() %{
+constraint(ALLOC_IN_RC(R0_regP));
+match(iRegP);
+format %{ %}
+interface(REG_INTER);
+%}
+operand R1RegP() %{
+constraint(ALLOC_IN_RC(R1_regP));
+match(iRegP);
+format %{ %}
+interface(REG_INTER);
+%}
+operand R2RegP() %{
+constraint(ALLOC_IN_RC(R2_regP));
+match(iRegP);
+format %{ %}
+interface(REG_INTER);
+%}
+operand RExceptionRegP() %{
+constraint(ALLOC_IN_RC(Rexception_regP));
+match(iRegP);
+format %{ %}
+interface(REG_INTER);
+%}
+operand RthreadRegP() %{
+constraint(ALLOC_IN_RC(Rthread_regP));
+match(iRegP);
+format %{ %}
+interface(REG_INTER);
+%}
+operand IPRegP() %{
+constraint(ALLOC_IN_RC(IP_regP));
+match(iRegP);
+format %{ %}
+interface(REG_INTER);
+%}
+operand LRRegP() %{
+constraint(ALLOC_IN_RC(LR_regP));
+match(iRegP);
+format %{ %}
+interface(REG_INTER);
+%}
+operand R0RegI() %{
+constraint(ALLOC_IN_RC(R0_regI));
+match(iRegI);
+format %{ %}
+interface(REG_INTER);
+%}
+operand R1RegI() %{
+constraint(ALLOC_IN_RC(R1_regI));
+match(iRegI);
+format %{ %}
+interface(REG_INTER);
+%}
+operand R2RegI() %{
+constraint(ALLOC_IN_RC(R2_regI));
+match(iRegI);
+format %{ %}
+interface(REG_INTER);
+%}
+operand R3RegI() %{
+constraint(ALLOC_IN_RC(R3_regI));
+match(iRegI);
+format %{ %}
+interface(REG_INTER);
+%}
+#ifndef AARCH64
+operand R12RegI() %{
+constraint(ALLOC_IN_RC(R12_regI));
+match(iRegI);
+format %{ %}
+interface(REG_INTER);
+%}
+#endif
+// Long Register
+operand iRegL() %{
+constraint(ALLOC_IN_RC(long_reg));
+match(RegL);
+#ifdef AARCH64
+match(iRegLd);
+#else
+match(R0R1RegL);
+match(R2R3RegL);
+#endif
+//match(iRegLex);
+format %{ %}
+interface(REG_INTER);
+%}
+operand iRegLd() %{
+constraint(ALLOC_IN_RC(long_reg_align));
+match(iRegL); // FIXME: allows unaligned R11/R12?
+format %{ %}
+interface(REG_INTER);
+%}
+#ifndef AARCH64
+// first long arg, or return value
+operand R0R1RegL() %{
+constraint(ALLOC_IN_RC(R0R1_regL));
+match(iRegL);
+format %{ %}
+interface(REG_INTER);
+%}
+operand R2R3RegL() %{
+constraint(ALLOC_IN_RC(R2R3_regL));
+match(iRegL);
+format %{ %}
+interface(REG_INTER);
+%}
+#endif
+// Condition Code Flag Register
+operand flagsReg() %{
+constraint(ALLOC_IN_RC(int_flags));
+match(RegFlags);
+format %{ "apsr" %}
+interface(REG_INTER);
+%}
+// Result of compare to 0 (TST)
+operand flagsReg_EQNELTGE() %{
+constraint(ALLOC_IN_RC(int_flags));
+match(RegFlags);
+format %{ "apsr_EQNELTGE" %}
+interface(REG_INTER);
+%}
+// Condition Code Register, unsigned comparisons.
+operand flagsRegU() %{
+constraint(ALLOC_IN_RC(int_flags));
+match(RegFlags);
+#ifdef TODO
+match(RegFlagsP);
+#endif
+format %{ "apsr_U" %}
+interface(REG_INTER);
+%}
+// Condition Code Register, pointer comparisons.
+operand flagsRegP() %{
+constraint(ALLOC_IN_RC(int_flags));
+match(RegFlags);
+format %{ "apsr_P" %}
+interface(REG_INTER);
+%}
+// Condition Code Register, long comparisons.
+#ifndef AARCH64
+operand flagsRegL_LTGE() %{
+constraint(ALLOC_IN_RC(int_flags));
+match(RegFlags);
+format %{ "apsr_L_LTGE" %}
+interface(REG_INTER);
+%}
+operand flagsRegL_EQNE() %{
+constraint(ALLOC_IN_RC(int_flags));
+match(RegFlags);
+format %{ "apsr_L_EQNE" %}
+interface(REG_INTER);
+%}
+operand flagsRegL_LEGT() %{
+constraint(ALLOC_IN_RC(int_flags));
+match(RegFlags);
+format %{ "apsr_L_LEGT" %}
+interface(REG_INTER);
+%}
+#endif
+// Condition Code Register, floating comparisons, unordered same as "less".
+operand flagsRegF() %{
+constraint(ALLOC_IN_RC(float_flags));
+match(RegFlags);
+format %{ "fpscr_F" %}
+interface(REG_INTER);
+%}
+// Vectors
+operand vecD() %{
+constraint(ALLOC_IN_RC(actual_dflt_reg));
+match(VecD);
+format %{ %}
+interface(REG_INTER);
+%}
+operand vecX() %{
+constraint(ALLOC_IN_RC(vectorx_reg));
+match(VecX);
+format %{ %}
+interface(REG_INTER);
+%}
+operand regD() %{
+constraint(ALLOC_IN_RC(actual_dflt_reg));
+match(RegD);
+match(regD_low);
+format %{ %}
+interface(REG_INTER);
+%}
+operand regF() %{
+constraint(ALLOC_IN_RC(sflt_reg));
+match(RegF);
+format %{ %}
+interface(REG_INTER);
+%}
+operand regD_low() %{
+constraint(ALLOC_IN_RC(dflt_low_reg));
+match(RegD);
+format %{ %}
+interface(REG_INTER);
+%}
+// Special Registers
+// Method Register
+operand inline_cache_regP(iRegP reg) %{
+constraint(ALLOC_IN_RC(Ricklass_regP));
+match(reg);
+format %{ %}
+interface(REG_INTER);
+%}
+operand interpreter_method_oop_regP(iRegP reg) %{
+constraint(ALLOC_IN_RC(Rmethod_regP));
+match(reg);
+format %{ %}
+interface(REG_INTER);
+%}
+//----------Complex Operands---------------------------------------------------
+// Indirect Memory Reference
+operand indirect(sp_ptr_RegP reg) %{
+constraint(ALLOC_IN_RC(sp_ptr_reg));
+match(reg);
+op_cost(100);
+format %{ "[$reg]" %}
+interface(MEMORY_INTER) %{
+base($reg);
+#ifdef AARCH64
+index(0xff); // 0xff => no index
+#else
+index(0xf); // PC => no index
+#endif
+scale(0x0);
+disp(0x0);
+%}
+%}
+#ifdef AARCH64
+// Indirect with scaled*1 uimm12 offset
+operand indOffsetU12ScaleB(sp_ptr_RegP reg, immUL12 offset) %{
+constraint(ALLOC_IN_RC(sp_ptr_reg));
+match(AddP reg offset);
+op_cost(100);
+format %{ "[$reg + $offset]" %}
+interface(MEMORY_INTER) %{
+base($reg);
+#ifdef AARCH64
+index(0xff); // 0xff => no index
+#else
+index(0xf); // PC => no index
+#endif
+scale(0x0);
+disp($offset);
+%}
+%}
+// Indirect with scaled*2 uimm12 offset
+operand indOffsetU12ScaleS(sp_ptr_RegP reg, immUL12x2 offset) %{
+constraint(ALLOC_IN_RC(sp_ptr_reg));
+match(AddP reg offset);
+op_cost(100);
+format %{ "[$reg + $offset]" %}
+interface(MEMORY_INTER) %{
+base($reg);
+#ifdef AARCH64
+index(0xff); // 0xff => no index
+#else
+index(0xf); // PC => no index
+#endif
+scale(0x0);
+disp($offset);
+%}
+%}
+// Indirect with scaled*4 uimm12 offset
+operand indOffsetU12ScaleI(sp_ptr_RegP reg, immUL12x4 offset) %{
+constraint(ALLOC_IN_RC(sp_ptr_reg));
+match(AddP reg offset);
+op_cost(100);
+format %{ "[$reg + $offset]" %}
+interface(MEMORY_INTER) %{
+base($reg);
+#ifdef AARCH64
+index(0xff); // 0xff => no index
+#else
+index(0xf); // PC => no index
+#endif
+scale(0x0);
+disp($offset);
+%}
+%}
+// Indirect with scaled*8 uimm12 offset
+operand indOffsetU12ScaleL(sp_ptr_RegP reg, immUL12x8 offset) %{
+constraint(ALLOC_IN_RC(sp_ptr_reg));
+match(AddP reg offset);
+op_cost(100);
+format %{ "[$reg + $offset]" %}
+interface(MEMORY_INTER) %{
+base($reg);
+#ifdef AARCH64
+index(0xff); // 0xff => no index
+#else
+index(0xf); // PC => no index
+#endif
+scale(0x0);
+disp($offset);
+%}
+%}
+// Indirect with scaled*16 uimm12 offset
+operand indOffsetU12ScaleQ(sp_ptr_RegP reg, immUL12x16 offset) %{
+constraint(ALLOC_IN_RC(sp_ptr_reg));
+match(AddP reg offset);
+op_cost(100);
+format %{ "[$reg + $offset]" %}
+interface(MEMORY_INTER) %{
+base($reg);
+#ifdef AARCH64
+index(0xff); // 0xff => no index
+#else
+index(0xf); // PC => no index
+#endif
+scale(0x0);
+disp($offset);
+%}
+%}
+#else // ! AARCH64
+// Indirect with Offset in ]-4096, 4096[
+operand indOffset12(sp_ptr_RegP reg, immI12 offset) %{
+constraint(ALLOC_IN_RC(sp_ptr_reg));
+match(AddP reg offset);
+op_cost(100);
+format %{ "[$reg + $offset]" %}
+interface(MEMORY_INTER) %{
+base($reg);
+#ifdef AARCH64
+index(0xff); // 0xff => no index
+#else
+index(0xf); // PC => no index
+#endif
+scale(0x0);
+disp($offset);
+%}
+%}
+// Indirect with offset for float load/store
+operand indOffsetFP(sp_ptr_RegP reg, immIFP offset) %{
+constraint(ALLOC_IN_RC(sp_ptr_reg));
+match(AddP reg offset);
+op_cost(100);
+format %{ "[$reg + $offset]" %}
+interface(MEMORY_INTER) %{
+base($reg);
+#ifdef AARCH64
+index(0xff); // 0xff => no index
+#else
+index(0xf); // PC => no index
+#endif
+scale(0x0);
+disp($offset);
+%}
+%}
+// Indirect with Offset for half and double words
+operand indOffsetHD(sp_ptr_RegP reg, immIHD offset) %{
+constraint(ALLOC_IN_RC(sp_ptr_reg));
+match(AddP reg offset);
+op_cost(100);
+format %{ "[$reg + $offset]" %}
+interface(MEMORY_INTER) %{
+base($reg);
+#ifdef AARCH64
+index(0xff); // 0xff => no index
+#else
+index(0xf); // PC => no index
+#endif
+scale(0x0);
+disp($offset);
+%}
+%}
+// Indirect with Offset and Offset+4 in ]-1024, 1024[
+operand indOffsetFPx2(sp_ptr_RegP reg, immX10x2 offset) %{
+constraint(ALLOC_IN_RC(sp_ptr_reg));
+match(AddP reg offset);
+op_cost(100);
+format %{ "[$reg + $offset]" %}
+interface(MEMORY_INTER) %{
+base($reg);
+#ifdef AARCH64
+index(0xff); // 0xff => no index
+#else
+index(0xf); // PC => no index
+#endif
+scale(0x0);
+disp($offset);
+%}
+%}
+// Indirect with Offset and Offset+4 in ]-4096, 4096[
+operand indOffset12x2(sp_ptr_RegP reg, immI12x2 offset) %{
+constraint(ALLOC_IN_RC(sp_ptr_reg));
+match(AddP reg offset);
+op_cost(100);
+format %{ "[$reg + $offset]" %}
+interface(MEMORY_INTER) %{
+base($reg);
+#ifdef AARCH64
+index(0xff); // 0xff => no index
+#else
+index(0xf); // PC => no index
+#endif
+scale(0x0);
+disp($offset);
+%}
+%}
+#endif // !AARCH64
+// Indirect with Register Index
+operand indIndex(iRegP addr, iRegX index) %{
+constraint(ALLOC_IN_RC(ptr_reg));
+match(AddP addr index);
+op_cost(100);
+format %{ "[$addr + $index]" %}
+interface(MEMORY_INTER) %{
+base($addr);
+index($index);
+scale(0x0);
+disp(0x0);
+%}
+%}
+#ifdef AARCH64
+// Indirect Memory Times Scale Plus Index Register
+operand indIndexScaleS(iRegP addr, iRegX index, immI_1 scale) %{
+constraint(ALLOC_IN_RC(ptr_reg));
+match(AddP addr (LShiftX index scale));
+op_cost(100);
+format %{"[$addr + $index << $scale]" %}
+interface(MEMORY_INTER) %{
+base($addr);
+index($index);
+scale($scale);
+disp(0x0);
+%}
+%}
+// Indirect Memory Times Scale Plus 32-bit Index Register
+operand indIndexIScaleS(iRegP addr, iRegI index, immI_1 scale) %{
+constraint(ALLOC_IN_RC(ptr_reg));
+match(AddP addr (LShiftX (ConvI2L index) scale));
+op_cost(100);
+format %{"[$addr + $index.w << $scale]" %}
+interface(MEMORY_INTER) %{
+base($addr);
+index($index);
+scale($scale);
+disp(0x7fffffff); // sxtw
+%}
+%}
+// Indirect Memory Times Scale Plus Index Register
+operand indIndexScaleI(iRegP addr, iRegX index, immI_2 scale) %{
+constraint(ALLOC_IN_RC(ptr_reg));
+match(AddP addr (LShiftX index scale));
+op_cost(100);
+format %{"[$addr + $index << $scale]" %}
+interface(MEMORY_INTER) %{
+base($addr);
+index($index);
+scale($scale);
+disp(0x0);
+%}
+%}
+// Indirect Memory Times Scale Plus 32-bit Index Register
+operand indIndexIScaleI(iRegP addr, iRegI index, immI_2 scale) %{
+constraint(ALLOC_IN_RC(ptr_reg));
+match(AddP addr (LShiftX (ConvI2L index) scale));
+op_cost(100);
+format %{"[$addr + $index.w << $scale]" %}
+interface(MEMORY_INTER) %{
+base($addr);
+index($index);
+scale($scale);
+disp(0x7fffffff); // sxtw
+%}
+%}
+// Indirect Memory Times Scale Plus Index Register
+operand indIndexScaleL(iRegP addr, iRegX index, immI_3 scale) %{
+constraint(ALLOC_IN_RC(ptr_reg));
+match(AddP addr (LShiftX index scale));
+op_cost(100);
+format %{"[$addr + $index << $scale]" %}
+interface(MEMORY_INTER) %{
+base($addr);
+index($index);
+scale($scale);
+disp(0x0);
+%}
+%}
+// Indirect Memory Times Scale Plus 32-bit Index Register
+operand indIndexIScaleL(iRegP addr, iRegI index, immI_3 scale) %{
+constraint(ALLOC_IN_RC(ptr_reg));
+match(AddP addr (LShiftX (ConvI2L index) scale));
+op_cost(100);
+format %{"[$addr + $index.w << $scale]" %}
+interface(MEMORY_INTER) %{
+base($addr);
+index($index);
+scale($scale);
+disp(0x7fffffff); // sxtw
+%}
+%}
+// Indirect Memory Times Scale Plus Index Register
+operand indIndexScaleQ(iRegP addr, iRegX index, immI_4 scale) %{
+constraint(ALLOC_IN_RC(ptr_reg));
+match(AddP addr (LShiftX index scale));
+op_cost(100);
+format %{"[$addr + $index << $scale]" %}
+interface(MEMORY_INTER) %{
+base($addr);
+index($index);
+scale($scale);
+disp(0x0);
+%}
+%}
+// Indirect Memory Times Scale Plus 32-bit Index Register
+operand indIndexIScaleQ(iRegP addr, iRegI index, immI_4 scale) %{
+constraint(ALLOC_IN_RC(ptr_reg));
+match(AddP addr (LShiftX (ConvI2L index) scale));
+op_cost(100);
+format %{"[$addr + $index.w << $scale]" %}
+interface(MEMORY_INTER) %{
+base($addr);
+index($index);
+scale($scale);
+disp(0x7fffffff); // sxtw
+%}
+%}
+#else
+// Indirect Memory Times Scale Plus Index Register
+operand indIndexScale(iRegP addr, iRegX index, immU5 scale) %{
+constraint(ALLOC_IN_RC(ptr_reg));
+match(AddP addr (LShiftX index scale));
+op_cost(100);
+format %{"[$addr + $index << $scale]" %}
+interface(MEMORY_INTER) %{
+base($addr);
+index($index);
+scale($scale);
+disp(0x0);
+%}
+%}
+#endif
+// Operands for expressing Control Flow
+// NOTE:  Label is a predefined operand which should not be redefined in
+//        the AD file.  It is generically handled within the ADLC.
+//----------Conditional Branch Operands----------------------------------------
+// Comparison Op  - This is the operation of the comparison, and is limited to
+//                  the following set of codes:
+//                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
+//
+// Other attributes of the comparison, such as unsignedness, are specified
+// by the comparison instruction that sets a condition code flags register.
+// That result is represented by a flags operand whose subtype is appropriate
+// to the unsignedness (etc.) of the comparison.
+//
+// Later, the instruction which matches both the Comparison Op (a Bool) and
+// the flags (produced by the Cmp) specifies the coding of the comparison op
+// by matching a specific subtype of Bool operand below, such as cmpOpU.
+operand cmpOp() %{
+match(Bool);
+format %{ "" %}
+interface(COND_INTER) %{
+equal(0x0);
+not_equal(0x1);
+less(0xb);
+greater_equal(0xa);
+less_equal(0xd);
+greater(0xc);
+overflow(0x0); // unsupported/unimplemented
+no_overflow(0x0); // unsupported/unimplemented
+%}
+%}
+// integer comparison with 0, signed
+operand cmpOp0() %{
+match(Bool);
+format %{ "" %}
+interface(COND_INTER) %{
+equal(0x0);
+not_equal(0x1);
+less(0x4);
+greater_equal(0x5);
+less_equal(0xd); // unsupported
+greater(0xc); // unsupported
+overflow(0x0); // unsupported/unimplemented
+no_overflow(0x0); // unsupported/unimplemented
+%}
+%}
+// Comparison Op, unsigned
+operand cmpOpU() %{
+match(Bool);
+format %{ "u" %}
+interface(COND_INTER) %{
+equal(0x0);
+not_equal(0x1);
+less(0x3);
+greater_equal(0x2);
+less_equal(0x9);
+greater(0x8);
+overflow(0x0); // unsupported/unimplemented
+no_overflow(0x0); // unsupported/unimplemented
+%}
+%}
+// Comparison Op, pointer (same as unsigned)
+operand cmpOpP() %{
+match(Bool);
+format %{ "p" %}
+interface(COND_INTER) %{
+equal(0x0);
+not_equal(0x1);
+less(0x3);
+greater_equal(0x2);
+less_equal(0x9);
+greater(0x8);
+overflow(0x0); // unsupported/unimplemented
+no_overflow(0x0); // unsupported/unimplemented
+%}
+%}
+operand cmpOpL() %{
+match(Bool);
+format %{ "L" %}
+interface(COND_INTER) %{
+equal(0x0);
+not_equal(0x1);
+less(0xb);
+greater_equal(0xa);
+less_equal(0xd);
+greater(0xc);
+overflow(0x0); // unsupported/unimplemented
+no_overflow(0x0); // unsupported/unimplemented
+%}
+%}
+operand cmpOpL_commute() %{
+match(Bool);
+format %{ "L" %}
+interface(COND_INTER) %{
+equal(0x0);
+not_equal(0x1);
+less(0xc);
+greater_equal(0xd);
+less_equal(0xa);
+greater(0xb);
+overflow(0x0); // unsupported/unimplemented
+no_overflow(0x0); // unsupported/unimplemented
+%}
+%}
+//----------OPERAND CLASSES----------------------------------------------------
+// Operand Classes are groups of operands that are used to simplify
+// instruction definitions by not requiring the AD writer to specify separate
+// instructions for every form of operand when the instruction accepts
+// multiple operand types with the same basic encoding and format.  The classic
+// case of this is memory operands.
+#ifdef AARCH64
+opclass memoryB(indirect, indIndex, indOffsetU12ScaleB);
+opclass memoryS(indirect, indIndex, indIndexScaleS, indIndexIScaleS, indOffsetU12ScaleS);
+opclass memoryI(indirect, indIndex, indIndexScaleI, indIndexIScaleI, indOffsetU12ScaleI);
+opclass memoryL(indirect, indIndex, indIndexScaleL, indIndexIScaleL, indOffsetU12ScaleL);
+opclass memoryP(indirect, indIndex, indIndexScaleL, indIndexIScaleL, indOffsetU12ScaleL);
+opclass memoryQ(indirect, indIndex, indIndexScaleQ, indIndexIScaleQ, indOffsetU12ScaleQ);
+opclass memoryF(indirect, indIndex, indIndexScaleI, indIndexIScaleI, indOffsetU12ScaleI);
+opclass memoryD(indirect, indIndex, indIndexScaleL, indIndexIScaleL, indOffsetU12ScaleL);
+opclass memoryScaledS(indIndexScaleS, indIndexIScaleS);
+opclass memoryScaledI(indIndexScaleI, indIndexIScaleI);
+opclass memoryScaledL(indIndexScaleL, indIndexIScaleL);
+opclass memoryScaledP(indIndexScaleL, indIndexIScaleL);
+opclass memoryScaledQ(indIndexScaleQ, indIndexIScaleQ);
+opclass memoryScaledF(indIndexScaleI, indIndexIScaleI);
+opclass memoryScaledD(indIndexScaleL, indIndexIScaleL);
+// when ldrex/strex is used:
+opclass memoryex ( indirect );
+opclass indIndexMemory( indIndex );
+opclass memoryvld ( indirect /* , write back mode not implemented */ );
+#else
+opclass memoryI ( indirect, indOffset12, indIndex, indIndexScale );
+opclass memoryP ( indirect, indOffset12, indIndex, indIndexScale );
+opclass memoryF ( indirect, indOffsetFP );
+opclass memoryF2 ( indirect, indOffsetFPx2 );
+opclass memoryD ( indirect, indOffsetFP );
+opclass memoryfp( indirect, indOffsetFP );
+opclass memoryB ( indirect, indIndex, indOffsetHD );
+opclass memoryS ( indirect, indIndex, indOffsetHD );
+opclass memoryL ( indirect, indIndex, indOffsetHD );
+opclass memoryScaledI(indIndexScale);
+opclass memoryScaledP(indIndexScale);
+// when ldrex/strex is used:
+opclass memoryex ( indirect );
+opclass indIndexMemory( indIndex );
+opclass memorylong ( indirect, indOffset12x2 );
+opclass memoryvld ( indirect /* , write back mode not implemented */ );
+#endif
+//----------PIPELINE-----------------------------------------------------------
+pipeline %{
+//----------ATTRIBUTES---------------------------------------------------------
+attributes %{
+fixed_size_instructions;           // Fixed size instructions
+max_instructions_per_bundle = 4;   // Up to 4 instructions per bundle
+instruction_unit_size = 4;         // An instruction is 4 bytes long
+instruction_fetch_unit_size = 16;  // The processor fetches one line
+instruction_fetch_units = 1;       // of 16 bytes
+// List of nop instructions
+nops( Nop_A0, Nop_A1, Nop_MS, Nop_FA, Nop_BR );
+%}
+//----------RESOURCES----------------------------------------------------------
+// Resources are the functional units available to the machine
+resources(A0, A1, MS, BR, FA, FM, IDIV, FDIV, IALU = A0 | A1);
+//----------PIPELINE DESCRIPTION-----------------------------------------------
+// Pipeline Description specifies the stages in the machine's pipeline
+pipe_desc(A, P, F, B, I, J, S, R, E, C, M, W, X, T, D);
+//----------PIPELINE CLASSES---------------------------------------------------
+// Pipeline Classes describe the stages in which input and output are
+// referenced by the hardware pipeline.
+// Integer ALU reg-reg operation
+pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
+single_instruction;
+dst   : E(write);
+src1  : R(read);
+src2  : R(read);
+IALU  : R;
+%}
+// Integer ALU reg-reg long operation
+pipe_class ialu_reg_reg_2(iRegL dst, iRegL src1, iRegL src2) %{
+instruction_count(2);
+dst   : E(write);
+src1  : R(read);
+src2  : R(read);
+IALU  : R;
+IALU  : R;
+%}
+// Integer ALU reg-reg long dependent operation
+pipe_class ialu_reg_reg_2_dep(iRegL dst, iRegL src1, iRegL src2, flagsReg cr) %{
+instruction_count(1); multiple_bundles;
+dst   : E(write);
+src1  : R(read);
+src2  : R(read);
+cr    : E(write);
+IALU  : R(2);
+%}
+// Integer ALU reg-imm operaion
+pipe_class ialu_reg_imm(iRegI dst, iRegI src1) %{
+single_instruction;
+dst   : E(write);
+src1  : R(read);
+IALU  : R;
+%}
+// Integer ALU reg-reg operation with condition code
+pipe_class ialu_cc_reg_reg(iRegI dst, iRegI src1, iRegI src2, flagsReg cr) %{
+single_instruction;
+dst   : E(write);
+cr    : E(write);
+src1  : R(read);
+src2  : R(read);
+IALU  : R;
+%}
+// Integer ALU zero-reg operation
+pipe_class ialu_zero_reg(iRegI dst, immI0 zero, iRegI src2) %{
+single_instruction;
+dst   : E(write);
+src2  : R(read);
+IALU  : R;
+%}
+// Integer ALU zero-reg operation with condition code only
+pipe_class ialu_cconly_zero_reg(flagsReg cr, iRegI src) %{
+single_instruction;
+cr    : E(write);
+src   : R(read);
+IALU  : R;
+%}
+// Integer ALU reg-reg operation with condition code only
+pipe_class ialu_cconly_reg_reg(flagsReg cr, iRegI src1, iRegI src2) %{
+single_instruction;
+cr    : E(write);
+src1  : R(read);
+src2  : R(read);
+IALU  : R;
+%}
+// Integer ALU reg-imm operation with condition code only
+pipe_class ialu_cconly_reg_imm(flagsReg cr, iRegI src1) %{
+single_instruction;
+cr    : E(write);
+src1  : R(read);
+IALU  : R;
+%}
+// Integer ALU reg-reg-zero operation with condition code only
+pipe_class ialu_cconly_reg_reg_zero(flagsReg cr, iRegI src1, iRegI src2, immI0 zero) %{
+single_instruction;
+cr    : E(write);
+src1  : R(read);
+src2  : R(read);
+IALU  : R;
+%}
+// Integer ALU reg-imm-zero operation with condition code only
+pipe_class ialu_cconly_reg_imm_zero(flagsReg cr, iRegI src1, immI0 zero) %{
+single_instruction;
+cr    : E(write);
+src1  : R(read);
+IALU  : R;
+%}
+// Integer ALU reg-reg operation with condition code, src1 modified
+pipe_class ialu_cc_rwreg_reg(flagsReg cr, iRegI src1, iRegI src2) %{
+single_instruction;
+cr    : E(write);
+src1  : E(write);
+src1  : R(read);
+src2  : R(read);
+IALU  : R;
+%}
+pipe_class cmpL_reg(iRegI dst, iRegL src1, iRegL src2, flagsReg cr ) %{
+multiple_bundles;
+dst   : E(write)+4;
+cr    : E(write);
+src1  : R(read);
+src2  : R(read);
+IALU  : R(3);
+BR    : R(2);
+%}
+// Integer ALU operation
+pipe_class ialu_none(iRegI dst) %{
+single_instruction;
+dst   : E(write);
+IALU  : R;
+%}
+// Integer ALU reg operation
+pipe_class ialu_reg(iRegI dst, iRegI src) %{
+single_instruction; may_have_no_code;
+dst   : E(write);
+src   : R(read);
+IALU  : R;
+%}
+// Integer ALU reg conditional operation
+// This instruction has a 1 cycle stall, and cannot execute
+// in the same cycle as the instruction setting the condition
+// code. We kludge this by pretending to read the condition code
+// 1 cycle earlier, and by marking the functional units as busy
+// for 2 cycles with the result available 1 cycle later than
+// is really the case.
+pipe_class ialu_reg_flags( iRegI op2_out, iRegI op2_in, iRegI op1, flagsReg cr ) %{
+single_instruction;
+op2_out : C(write);
+op1     : R(read);
+cr      : R(read);       // This is really E, with a 1 cycle stall
+BR      : R(2);
+MS      : R(2);
+%}
+// Integer ALU reg operation
+pipe_class ialu_move_reg_L_to_I(iRegI dst, iRegL src) %{
+single_instruction; may_have_no_code;
+dst   : E(write);
+src   : R(read);
+IALU  : R;
+%}
+pipe_class ialu_move_reg_I_to_L(iRegL dst, iRegI src) %{
+single_instruction; may_have_no_code;
+dst   : E(write);
+src   : R(read);
+IALU  : R;
+%}
+// Two integer ALU reg operations
+pipe_class ialu_reg_2(iRegL dst, iRegL src) %{
+instruction_count(2);
+dst   : E(write);
+src   : R(read);
+A0    : R;
+A1    : R;
+%}
+// Two integer ALU reg operations
+pipe_class ialu_move_reg_L_to_L(iRegL dst, iRegL src) %{
+instruction_count(2); may_have_no_code;
+dst   : E(write);
+src   : R(read);
+A0    : R;
+A1    : R;
+%}
+// Integer ALU imm operation
+pipe_class ialu_imm(iRegI dst) %{
+single_instruction;
+dst   : E(write);
+IALU  : R;
+%}
+pipe_class ialu_imm_n(iRegI dst) %{
+single_instruction;
+dst   : E(write);
+IALU  : R;
+%}
+// Integer ALU reg-reg with carry operation
+pipe_class ialu_reg_reg_cy(iRegI dst, iRegI src1, iRegI src2, iRegI cy) %{
+single_instruction;
+dst   : E(write);
+src1  : R(read);
+src2  : R(read);
+IALU  : R;
+%}
+// Integer ALU cc operation
+pipe_class ialu_cc(iRegI dst, flagsReg cc) %{
+single_instruction;
+dst   : E(write);
+cc    : R(read);
+IALU  : R;
+%}
+// Integer ALU cc / second IALU operation
+pipe_class ialu_reg_ialu( iRegI dst, iRegI src ) %{
+instruction_count(1); multiple_bundles;
+dst   : E(write)+1;
+src   : R(read);
+IALU  : R;
+%}
+// Integer ALU cc / second IALU operation
+pipe_class ialu_reg_reg_ialu( iRegI dst, iRegI p, iRegI q ) %{
+instruction_count(1); multiple_bundles;
+dst   : E(write)+1;
+p     : R(read);
+q     : R(read);
+IALU  : R;
+%}
+// Integer ALU hi-lo-reg operation
+pipe_class ialu_hi_lo_reg(iRegI dst, immI src) %{
+instruction_count(1); multiple_bundles;
+dst   : E(write)+1;
+IALU  : R(2);
+%}
+// Long Constant
+pipe_class loadConL( iRegL dst, immL src ) %{
+instruction_count(2); multiple_bundles;
+dst   : E(write)+1;
+IALU  : R(2);
+IALU  : R(2);
+%}
+// Pointer Constant
+pipe_class loadConP( iRegP dst, immP src ) %{
+instruction_count(0); multiple_bundles;
+fixed_latency(6);
+%}
+// Polling Address
+pipe_class loadConP_poll( iRegP dst, immP_poll src ) %{
+dst   : E(write);
+IALU  : R;
+%}
+// Long Constant small
+pipe_class loadConLlo( iRegL dst, immL src ) %{
+instruction_count(2);
+dst   : E(write);
+IALU  : R;
+IALU  : R;
+%}
+// [PHH] This is wrong for 64-bit.  See LdImmF/D.
+pipe_class loadConFD(regF dst, immF src, iRegP tmp) %{
+instruction_count(1); multiple_bundles;
+src   : R(read);
+dst   : M(write)+1;
+IALU  : R;
+MS    : E;
+%}
+// Integer ALU nop operation
+pipe_class ialu_nop() %{
+single_instruction;
+IALU  : R;
+%}
+// Integer ALU nop operation
+pipe_class ialu_nop_A0() %{
+single_instruction;
+A0    : R;
+%}
+// Integer ALU nop operation
+pipe_class ialu_nop_A1() %{
+single_instruction;
+A1    : R;
+%}
+// Integer Multiply reg-reg operation
+pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
+single_instruction;
+dst   : E(write);
+src1  : R(read);
+src2  : R(read);
+MS    : R(5);
+%}
+pipe_class mulL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
+single_instruction;
+dst   : E(write)+4;
+src1  : R(read);
+src2  : R(read);
+MS    : R(6);
+%}
+// Integer Divide reg-reg
+pipe_class sdiv_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI temp, flagsReg cr) %{
+instruction_count(1); multiple_bundles;
+dst   : E(write);
+temp  : E(write);
+src1  : R(read);
+src2  : R(read);
+temp  : R(read);
+MS    : R(38);
+%}
+// Long Divide
+pipe_class divL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
+dst  : E(write)+71;
+src1 : R(read);
+src2 : R(read)+1;
+MS   : R(70);
+%}
+// Floating Point Add Float
+pipe_class faddF_reg_reg(regF dst, regF src1, regF src2) %{
+single_instruction;
+dst   : X(write);
+src1  : E(read);
+src2  : E(read);
+FA    : R;
+%}
+// Floating Point Add Double
+pipe_class faddD_reg_reg(regD dst, regD src1, regD src2) %{
+single_instruction;
+dst   : X(write);
+src1  : E(read);
+src2  : E(read);
+FA    : R;
+%}
+// Floating Point Conditional Move based on integer flags
+pipe_class int_conditional_float_move (cmpOp cmp, flagsReg cr, regF dst, regF src) %{
+single_instruction;
+dst   : X(write);
+src   : E(read);
+cr    : R(read);
+FA    : R(2);
+BR    : R(2);
+%}
+// Floating Point Conditional Move based on integer flags
+pipe_class int_conditional_double_move (cmpOp cmp, flagsReg cr, regD dst, regD src) %{
+single_instruction;
+dst   : X(write);
+src   : E(read);
+cr    : R(read);
+FA    : R(2);
+BR    : R(2);
+%}
+// Floating Point Multiply Float
+pipe_class fmulF_reg_reg(regF dst, regF src1, regF src2) %{
+single_instruction;
+dst   : X(write);
+src1  : E(read);
+src2  : E(read);
+FM    : R;
+%}
+// Floating Point Multiply Double
+pipe_class fmulD_reg_reg(regD dst, regD src1, regD src2) %{
+single_instruction;
+dst   : X(write);
+src1  : E(read);
+src2  : E(read);
+FM    : R;
+%}
+// Floating Point Divide Float
+pipe_class fdivF_reg_reg(regF dst, regF src1, regF src2) %{
+single_instruction;
+dst   : X(write);
+src1  : E(read);
+src2  : E(read);
+FM    : R;
+FDIV  : C(14);
+%}
+// Floating Point Divide Double
+pipe_class fdivD_reg_reg(regD dst, regD src1, regD src2) %{
+single_instruction;
+dst   : X(write);
+src1  : E(read);
+src2  : E(read);
+FM    : R;
+FDIV  : C(17);
+%}
+// Floating Point Move/Negate/Abs Float
+pipe_class faddF_reg(regF dst, regF src) %{
+single_instruction;
+dst   : W(write);
+src   : E(read);
+FA    : R(1);
+%}
+// Floating Point Move/Negate/Abs Double
+pipe_class faddD_reg(regD dst, regD src) %{
+single_instruction;
+dst   : W(write);
+src   : E(read);
+FA    : R;
+%}
+// Floating Point Convert F->D
+pipe_class fcvtF2D(regD dst, regF src) %{
+single_instruction;
+dst   : X(write);
+src   : E(read);
+FA    : R;
+%}
+// Floating Point Convert I->D
+pipe_class fcvtI2D(regD dst, regF src) %{
+single_instruction;
+dst   : X(write);
+src   : E(read);
+FA    : R;
+%}
+// Floating Point Convert LHi->D
+pipe_class fcvtLHi2D(regD dst, regD src) %{
+single_instruction;
+dst   : X(write);
+src   : E(read);
+FA    : R;
+%}
+// Floating Point Convert L->D
+pipe_class fcvtL2D(regD dst, iRegL src) %{
+single_instruction;
+dst   : X(write);
+src   : E(read);
+FA    : R;
+%}
+// Floating Point Convert L->F
+pipe_class fcvtL2F(regF dst, iRegL src) %{
+single_instruction;
+dst   : X(write);
+src   : E(read);
+FA    : R;
+%}
+// Floating Point Convert D->F
+pipe_class fcvtD2F(regD dst, regF src) %{
+single_instruction;
+dst   : X(write);
+src   : E(read);
+FA    : R;
+%}
+// Floating Point Convert I->L
+pipe_class fcvtI2L(regD dst, regF src) %{
+single_instruction;
+dst   : X(write);
+src   : E(read);
+FA    : R;
+%}
+// Floating Point Convert D->F
+pipe_class fcvtD2I(iRegI dst, regD src, flagsReg cr) %{
+instruction_count(1); multiple_bundles;
+dst   : X(write)+6;
+src   : E(read);
+FA    : R;
+%}
+// Floating Point Convert D->L
+pipe_class fcvtD2L(regD dst, regD src, flagsReg cr) %{
+instruction_count(1); multiple_bundles;
+dst   : X(write)+6;
+src   : E(read);
+FA    : R;
+%}
+// Floating Point Convert F->I
+pipe_class fcvtF2I(regF dst, regF src, flagsReg cr) %{
+instruction_count(1); multiple_bundles;
+dst   : X(write)+6;
+src   : E(read);
+FA    : R;
+%}
+// Floating Point Convert F->L
+pipe_class fcvtF2L(regD dst, regF src, flagsReg cr) %{
+instruction_count(1); multiple_bundles;
+dst   : X(write)+6;
+src   : E(read);
+FA    : R;
+%}
+// Floating Point Convert I->F
+pipe_class fcvtI2F(regF dst, regF src) %{
+single_instruction;
+dst   : X(write);
+src   : E(read);
+FA    : R;
+%}
+// Floating Point Compare
+pipe_class faddF_fcc_reg_reg_zero(flagsRegF cr, regF src1, regF src2, immI0 zero) %{
+single_instruction;
+cr    : X(write);
+src1  : E(read);
+src2  : E(read);
+FA    : R;
+%}
+// Floating Point Compare
+pipe_class faddD_fcc_reg_reg_zero(flagsRegF cr, regD src1, regD src2, immI0 zero) %{
+single_instruction;
+cr    : X(write);
+src1  : E(read);
+src2  : E(read);
+FA    : R;
+%}
+// Floating Add Nop
+pipe_class fadd_nop() %{
+single_instruction;
+FA  : R;
+%}
+// Integer Store to Memory
+pipe_class istore_mem_reg(memoryI mem, iRegI src) %{
+single_instruction;
+mem   : R(read);
+src   : C(read);
+MS    : R;
+%}
+// Integer Store to Memory
+pipe_class istore_mem_spORreg(memoryI mem, sp_ptr_RegP src) %{
+single_instruction;
+mem   : R(read);
+src   : C(read);
+MS    : R;
+%}
+// Float Store
+pipe_class fstoreF_mem_reg(memoryF mem, RegF src) %{
+single_instruction;
+mem : R(read);
+src : C(read);
+MS  : R;
+%}
+// Float Store
+pipe_class fstoreF_mem_zero(memoryF mem, immF0 src) %{
+single_instruction;
+mem : R(read);
+MS  : R;
+%}
+// Double Store
+pipe_class fstoreD_mem_reg(memoryD mem, RegD src) %{
+instruction_count(1);
+mem : R(read);
+src : C(read);
+MS  : R;
+%}
+// Double Store
+pipe_class fstoreD_mem_zero(memoryD mem, immD0 src) %{
+single_instruction;
+mem : R(read);
+MS  : R;
+%}
+// Integer Load (when sign bit propagation not needed)
+pipe_class iload_mem(iRegI dst, memoryI mem) %{
+single_instruction;
+mem : R(read);
+dst : C(write);
+MS  : R;
+%}
+// Integer Load (when sign bit propagation or masking is needed)
+pipe_class iload_mask_mem(iRegI dst, memoryI mem) %{
+single_instruction;
+mem : R(read);
+dst : M(write);
+MS  : R;
+%}
+// Float Load
+pipe_class floadF_mem(regF dst, memoryF mem) %{
+single_instruction;
+mem : R(read);
+dst : M(write);
+MS  : R;
+%}
+// Float Load
+pipe_class floadD_mem(regD dst, memoryD mem) %{
+instruction_count(1); multiple_bundles; // Again, unaligned argument is only multiple case
+mem : R(read);
+dst : M(write);
+MS  : R;
+%}
+// Memory Nop
+pipe_class mem_nop() %{
+single_instruction;
+MS  : R;
+%}
+pipe_class sethi(iRegP dst, immI src) %{
+single_instruction;
+dst  : E(write);
+IALU : R;
+%}
+pipe_class loadPollP(iRegP poll) %{
+single_instruction;
+poll : R(read);
+MS   : R;
+%}
+pipe_class br(Universe br, label labl) %{
+single_instruction_with_delay_slot;
+BR  : R;
+%}
+pipe_class br_cc(Universe br, cmpOp cmp, flagsReg cr, label labl) %{
+single_instruction_with_delay_slot;
+cr    : E(read);
+BR    : R;
+%}
+pipe_class br_reg(Universe br, cmpOp cmp, iRegI op1, label labl) %{
+single_instruction_with_delay_slot;
+op1 : E(read);
+BR  : R;
+MS  : R;
+%}
+pipe_class br_nop() %{
+single_instruction;
+BR  : R;
+%}
+pipe_class simple_call(method meth) %{
+instruction_count(2); multiple_bundles; force_serialization;
+fixed_latency(100);
+BR  : R(1);
+MS  : R(1);
+A0  : R(1);
+%}
+pipe_class compiled_call(method meth) %{
+instruction_count(1); multiple_bundles; force_serialization;
+fixed_latency(100);
+MS  : R(1);
+%}
+pipe_class call(method meth) %{
+instruction_count(0); multiple_bundles; force_serialization;
+fixed_latency(100);
+%}
+pipe_class tail_call(Universe ignore, label labl) %{
+single_instruction; has_delay_slot;
+fixed_latency(100);
+BR  : R(1);
+MS  : R(1);
+%}
+pipe_class ret(Universe ignore) %{
+single_instruction; has_delay_slot;
+BR  : R(1);
+MS  : R(1);
+%}
+// The real do-nothing guy
+pipe_class empty( ) %{
+instruction_count(0);
+%}
+pipe_class long_memory_op() %{
+instruction_count(0); multiple_bundles; force_serialization;
+fixed_latency(25);
+MS  : R(1);
+%}
+// Check-cast
+pipe_class partial_subtype_check_pipe(Universe ignore, iRegP array, iRegP match ) %{
+array : R(read);
+match  : R(read);
+IALU   : R(2);
+BR     : R(2);
+MS     : R;
+%}
+// Convert FPU flags into +1,0,-1
+pipe_class floating_cmp( iRegI dst, regF src1, regF src2 ) %{
+src1  : E(read);
+src2  : E(read);
+dst   : E(write);
+FA    : R;
+MS    : R(2);
+BR    : R(2);
+%}
+// Compare for p < q, and conditionally add y
+pipe_class cadd_cmpltmask( iRegI p, iRegI q, iRegI y ) %{
+p     : E(read);
+q     : E(read);
+y     : E(read);
+IALU  : R(3)
+%}
+// Perform a compare, then move conditionally in a branch delay slot.
+pipe_class min_max( iRegI src2, iRegI srcdst ) %{
+src2   : E(read);
+srcdst : E(read);
+IALU   : R;
+BR     : R;
+%}
+// Define the class for the Nop node
+define %{
+MachNop = ialu_nop;
+%}
+%}
+//----------INSTRUCTIONS-------------------------------------------------------
+//------------Special Nop instructions for bundling - no match rules-----------
+// Nop using the A0 functional unit
+instruct Nop_A0() %{
+ins_pipe(ialu_nop_A0);
+%}
+// Nop using the A1 functional unit
+instruct Nop_A1( ) %{
+ins_pipe(ialu_nop_A1);
+%}
+// Nop using the memory functional unit
+instruct Nop_MS( ) %{
+ins_pipe(mem_nop);
+%}
+// Nop using the floating add functional unit
+instruct Nop_FA( ) %{
+ins_pipe(fadd_nop);
+%}
+// Nop using the branch functional unit
+instruct Nop_BR( ) %{
+ins_pipe(br_nop);
+%}
+//----------Load/Store/Move Instructions---------------------------------------
+//----------Load Instructions--------------------------------------------------
+// Load Byte (8bit signed)
+instruct loadB(iRegI dst, memoryB mem) %{
+match(Set dst (LoadB mem));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "LDRSB   $dst,$mem\t! byte -> int" %}
+ins_encode %{
+// High 32 bits are harmlessly set on Aarch64
+__ ldrsb($dst$$Register, $mem$$Address);
+%}
+ins_pipe(iload_mask_mem);
+%}
+// Load Byte (8bit signed) into a Long Register
+instruct loadB2L(iRegL dst, memoryB mem) %{
+match(Set dst (ConvI2L (LoadB mem)));
+ins_cost(MEMORY_REF_COST);
+#ifdef AARCH64
+size(4);
+format %{ "LDRSB $dst,$mem\t! byte -> long"  %}
+ins_encode %{
+__ ldrsb($dst$$Register, $mem$$Address);
+%}
+#else
+size(8);
+format %{ "LDRSB $dst.lo,$mem\t! byte -> long\n\t"
+"ASR   $dst.hi,$dst.lo,31" %}
+ins_encode %{
+__ ldrsb($dst$$Register, $mem$$Address);
+__ mov($dst$$Register->successor(), AsmOperand($dst$$Register, asr, 31));
+%}
+#endif
+ins_pipe(iload_mask_mem);
+%}
+// Load Unsigned Byte (8bit UNsigned) into an int reg
+instruct loadUB(iRegI dst, memoryB mem) %{
+match(Set dst (LoadUB mem));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "LDRB   $dst,$mem\t! ubyte -> int" %}
+ins_encode %{
+__ ldrb($dst$$Register, $mem$$Address);
+%}
+ins_pipe(iload_mem);
+%}
+// Load Unsigned Byte (8bit UNsigned) into a Long Register
+instruct loadUB2L(iRegL dst, memoryB mem) %{
+match(Set dst (ConvI2L (LoadUB mem)));
+ins_cost(MEMORY_REF_COST);
+#ifdef AARCH64
+size(4);
+format %{ "LDRB  $dst,$mem\t! ubyte -> long"  %}
+ins_encode %{
+__ ldrb($dst$$Register, $mem$$Address);
+%}
+#else
+size(8);
+format %{ "LDRB  $dst.lo,$mem\t! ubyte -> long\n\t"
+"MOV   $dst.hi,0" %}
+ins_encode %{
+__ ldrb($dst$$Register, $mem$$Address);
+__ mov($dst$$Register->successor(), 0);
+%}
+#endif
+ins_pipe(iload_mem);
+%}
+// Load Unsigned Byte (8 bit UNsigned) with immediate mask into Long Register
+instruct loadUB2L_limmI(iRegL dst, memoryB mem, limmIlow8 mask) %{
+match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
+#ifdef AARCH64
+ins_cost(MEMORY_REF_COST + DEFAULT_COST);
+size(8);
+format %{ "LDRB  $dst,$mem\t! ubyte -> long\n\t"
+"AND  $dst,$dst,$mask" %}
+ins_encode %{
+__ ldrb($dst$$Register, $mem$$Address);
+__ andr($dst$$Register, $dst$$Register, limmI_low($mask$$constant, 8));
+%}
+#else
+ins_cost(MEMORY_REF_COST + 2*DEFAULT_COST);
+size(12);
+format %{ "LDRB  $dst.lo,$mem\t! ubyte -> long\n\t"
+"MOV   $dst.hi,0\n\t"
+"AND  $dst.lo,$dst.lo,$mask" %}
+ins_encode %{
+__ ldrb($dst$$Register, $mem$$Address);
+__ mov($dst$$Register->successor(), 0);
+__ andr($dst$$Register, $dst$$Register, limmI_low($mask$$constant, 8));
+%}
+#endif
+ins_pipe(iload_mem);
+%}
+// Load Short (16bit signed)
+#ifdef AARCH64
+// XXX This variant shouldn't be necessary if 6217251 is implemented
+instruct loadSoff(iRegI dst, memoryScaledS mem, aimmX off, iRegP tmp) %{
+match(Set dst (LoadS (AddP mem off)));
+ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
+effect(TEMP tmp);
+size(4 * 2);
+format %{ "LDRSH   $dst,$mem+$off\t! short temp=$tmp" %}
+ins_encode %{
+Register base = reg_to_register_object($mem$$base);
+__ add($tmp$$Register, base, $off$$constant);
+Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
+__ ldrsh($dst$$Register, nmem);
+%}
+ins_pipe(iload_mask_mem);
+%}
+#endif
+instruct loadS(iRegI dst, memoryS mem) %{
+match(Set dst (LoadS mem));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "LDRSH   $dst,$mem\t! short" %}
+ins_encode %{
+__ ldrsh($dst$$Register, $mem$$Address);
+%}
+ins_pipe(iload_mask_mem);
+%}
+// Load Short (16 bit signed) to Byte (8 bit signed)
+instruct loadS2B(iRegI dst, memoryS mem, immI_24 twentyfour) %{
+match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "LDRSB   $dst,$mem\t! short -> byte" %}
+ins_encode %{
+// High 32 bits are harmlessly set on Aarch64
+__ ldrsb($dst$$Register, $mem$$Address);
+%}
+ins_pipe(iload_mask_mem);
+%}
+// Load Short (16bit signed) into a Long Register
+instruct loadS2L(iRegL dst, memoryS mem) %{
+match(Set dst (ConvI2L (LoadS mem)));
+ins_cost(MEMORY_REF_COST);
+#ifdef AARCH64
+size(4);
+format %{ "LDRSH $dst,$mem\t! short -> long"  %}
+ins_encode %{
+__ ldrsh($dst$$Register, $mem$$Address);
+%}
+#else
+size(8);
+format %{ "LDRSH $dst.lo,$mem\t! short -> long\n\t"
+"ASR   $dst.hi,$dst.lo,31" %}
+ins_encode %{
+__ ldrsh($dst$$Register, $mem$$Address);
+__ mov($dst$$Register->successor(), AsmOperand($dst$$Register, asr, 31));
+%}
+#endif
+ins_pipe(iload_mask_mem);
+%}
+// Load Unsigned Short/Char (16bit UNsigned)
+#ifdef AARCH64
+// XXX This variant shouldn't be necessary if 6217251 is implemented
+instruct loadUSoff(iRegI dst, memoryScaledS mem, aimmX off, iRegP tmp) %{
+match(Set dst (LoadUS (AddP mem off)));
+ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
+effect(TEMP tmp);
+size(4 * 2);
+format %{ "LDRH   $dst,$mem+$off\t! ushort/char temp=$tmp" %}
+ins_encode %{
+Register base = reg_to_register_object($mem$$base);
+__ add($tmp$$Register, base, $off$$constant);
+Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
+__ ldrh($dst$$Register, nmem);
+%}
+ins_pipe(iload_mem);
+%}
+#endif
+instruct loadUS(iRegI dst, memoryS mem) %{
+match(Set dst (LoadUS mem));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "LDRH   $dst,$mem\t! ushort/char" %}
+ins_encode %{
+__ ldrh($dst$$Register, $mem$$Address);
+%}
+ins_pipe(iload_mem);
+%}
+// Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
+instruct loadUS2B(iRegI dst, memoryB mem, immI_24 twentyfour) %{
+match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "LDRSB   $dst,$mem\t! ushort -> byte" %}
+ins_encode %{
+__ ldrsb($dst$$Register, $mem$$Address);
+%}
+ins_pipe(iload_mask_mem);
+%}
+// Load Unsigned Short/Char (16bit UNsigned) into a Long Register
+instruct loadUS2L(iRegL dst, memoryS mem) %{
+match(Set dst (ConvI2L (LoadUS mem)));
+ins_cost(MEMORY_REF_COST);
+#ifdef AARCH64
+size(4);
+format %{ "LDRH  $dst,$mem\t! short -> long"  %}
+ins_encode %{
+__ ldrh($dst$$Register, $mem$$Address);
+%}
+#else
+size(8);
+format %{ "LDRH  $dst.lo,$mem\t! short -> long\n\t"
+"MOV   $dst.hi, 0" %}
+ins_encode %{
+__ ldrh($dst$$Register, $mem$$Address);
+__ mov($dst$$Register->successor(), 0);
+%}
+#endif
+ins_pipe(iload_mem);
+%}
+// Load Unsigned Short/Char (16bit UNsigned) with mask 0xFF into a Long Register
+instruct loadUS2L_immI_255(iRegL dst, memoryB mem, immI_255 mask) %{
+match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
+ins_cost(MEMORY_REF_COST);
+#ifdef AARCH64
+size(4);
+format %{ "LDRB  $dst,$mem"  %}
+ins_encode %{
+__ ldrb($dst$$Register, $mem$$Address);
+%}
+#else
+size(8);
+format %{ "LDRB  $dst.lo,$mem\t! \n\t"
+"MOV   $dst.hi, 0" %}
+ins_encode %{
+__ ldrb($dst$$Register, $mem$$Address);
+__ mov($dst$$Register->successor(), 0);
+%}
+#endif
+ins_pipe(iload_mem);
+%}
+// Load Unsigned Short/Char (16bit UNsigned) with a immediate mask into a Long Register
+instruct loadUS2L_limmI(iRegL dst, memoryS mem, limmI mask) %{
+match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
+#ifdef AARCH64
+ins_cost(MEMORY_REF_COST + 1*DEFAULT_COST);
+size(8);
+format %{ "LDRH   $dst,$mem\t! ushort/char & mask -> long\n\t"
+"AND    $dst,$dst,$mask" %}
+ins_encode %{
+__ ldrh($dst$$Register, $mem$$Address);
+__ andr($dst$$Register, $dst$$Register, (uintx)$mask$$constant);
+%}
+#else
+ins_cost(MEMORY_REF_COST + 2*DEFAULT_COST);
+size(12);
+format %{ "LDRH   $dst,$mem\t! ushort/char & mask -> long\n\t"
+"MOV    $dst.hi, 0\n\t"
+"AND    $dst,$dst,$mask" %}
+ins_encode %{
+__ ldrh($dst$$Register, $mem$$Address);
+__ mov($dst$$Register->successor(), 0);
+__ andr($dst$$Register, $dst$$Register, $mask$$constant);
+%}
+#endif
+ins_pipe(iload_mem);
+%}
+// Load Integer
+#ifdef AARCH64
+// XXX This variant shouldn't be necessary if 6217251 is implemented
+instruct loadIoff(iRegI dst, memoryScaledI mem, aimmX off, iRegP tmp) %{
+match(Set dst (LoadI (AddP mem off)));
+ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
+effect(TEMP tmp);
+size(4 * 2);
+format %{ "ldr_s32 $dst,$mem+$off\t! int temp=$tmp" %}
+ins_encode %{
+Register base = reg_to_register_object($mem$$base);
+__ add($tmp$$Register, base, $off$$constant);
+Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
+__ ldr_s32($dst$$Register, nmem);
+%}
+ins_pipe(iload_mem);
+%}
+#endif
+instruct loadI(iRegI dst, memoryI mem) %{
+match(Set dst (LoadI mem));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "ldr_s32 $dst,$mem\t! int" %}
+ins_encode %{
+__ ldr_s32($dst$$Register, $mem$$Address);
+%}
+ins_pipe(iload_mem);
+%}
+// Load Integer to Byte (8 bit signed)
+instruct loadI2B(iRegI dst, memoryS mem, immI_24 twentyfour) %{
+match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "LDRSB   $dst,$mem\t! int -> byte" %}
+ins_encode %{
+__ ldrsb($dst$$Register, $mem$$Address);
+%}
+ins_pipe(iload_mask_mem);
+%}
+// Load Integer to Unsigned Byte (8 bit UNsigned)
+instruct loadI2UB(iRegI dst, memoryB mem, immI_255 mask) %{
+match(Set dst (AndI (LoadI mem) mask));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "LDRB   $dst,$mem\t! int -> ubyte" %}
+ins_encode %{
+__ ldrb($dst$$Register, $mem$$Address);
+%}
+ins_pipe(iload_mask_mem);
+%}
+// Load Integer to Short (16 bit signed)
+instruct loadI2S(iRegI dst, memoryS mem, immI_16 sixteen) %{
+match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "LDRSH   $dst,$mem\t! int -> short" %}
+ins_encode %{
+__ ldrsh($dst$$Register, $mem$$Address);
+%}
+ins_pipe(iload_mask_mem);
+%}
+// Load Integer to Unsigned Short (16 bit UNsigned)
+instruct loadI2US(iRegI dst, memoryS mem, immI_65535 mask) %{
+match(Set dst (AndI (LoadI mem) mask));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "LDRH   $dst,$mem\t! int -> ushort/char" %}
+ins_encode %{
+__ ldrh($dst$$Register, $mem$$Address);
+%}
+ins_pipe(iload_mask_mem);
+%}
+// Load Integer into a Long Register
+instruct loadI2L(iRegL dst, memoryI mem) %{
+match(Set dst (ConvI2L (LoadI mem)));
+#ifdef AARCH64
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "LDRSW $dst.lo,$mem\t! int -> long"  %}
+ins_encode %{
+__ ldr_s32($dst$$Register, $mem$$Address);
+%}
+#else
+ins_cost(MEMORY_REF_COST);
+size(8);
+format %{ "LDR   $dst.lo,$mem\t! int -> long\n\t"
+"ASR   $dst.hi,$dst.lo,31\t! int->long" %}
+ins_encode %{
+__ ldr($dst$$Register, $mem$$Address);
+__ mov($dst$$Register->successor(), AsmOperand($dst$$Register, asr, 31));
+%}
+#endif
+ins_pipe(iload_mask_mem);
+%}
+// Load Integer with mask 0xFF into a Long Register
+instruct loadI2L_immI_255(iRegL dst, memoryB mem, immI_255 mask) %{
+match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
+#ifdef AARCH64
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "LDRB   $dst.lo,$mem\t! int & 0xFF -> long"  %}
+ins_encode %{
+__ ldrb($dst$$Register, $mem$$Address);
+%}
+#else
+ins_cost(MEMORY_REF_COST);
+size(8);
+format %{ "LDRB   $dst.lo,$mem\t! int & 0xFF -> long\n\t"
+"MOV    $dst.hi, 0" %}
+ins_encode %{
+__ ldrb($dst$$Register, $mem$$Address);
+__ mov($dst$$Register->successor(), 0);
+%}
+#endif
+ins_pipe(iload_mem);
+%}
+// Load Integer with mask 0xFFFF into a Long Register
+instruct loadI2L_immI_65535(iRegL dst, memoryS mem, immI_65535 mask) %{
+match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
+ins_cost(MEMORY_REF_COST);
+#ifdef AARCH64
+size(4);
+format %{ "LDRH   $dst,$mem\t! int & 0xFFFF -> long" %}
+ins_encode %{
+__ ldrh($dst$$Register, $mem$$Address);
+%}
+#else
+size(8);
+format %{ "LDRH   $dst,$mem\t! int & 0xFFFF -> long\n\t"
+"MOV    $dst.hi, 0" %}
+ins_encode %{
+__ ldrh($dst$$Register, $mem$$Address);
+__ mov($dst$$Register->successor(), 0);
+%}
+#endif
+ins_pipe(iload_mask_mem);
+%}
+#ifdef AARCH64
+// Load Integer with an immediate mask into a Long Register
+instruct loadI2L_limmI(iRegL dst, memoryI mem, limmI mask) %{
+match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
+ins_cost(MEMORY_REF_COST + 1*DEFAULT_COST);
+size(8);
+format %{ "LDRSW $dst,$mem\t! int -> long\n\t"
+"AND   $dst,$dst,$mask" %}
+ins_encode %{
+__ ldr_s32($dst$$Register, $mem$$Address);
+__ andr($dst$$Register, $dst$$Register, (uintx)$mask$$constant);
+%}
+ins_pipe(iload_mem);
+%}
+#else
+// Load Integer with a 31-bit immediate mask into a Long Register
+instruct loadI2L_limmU31(iRegL dst, memoryI mem, limmU31 mask) %{
+match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
+ins_cost(MEMORY_REF_COST + 2*DEFAULT_COST);
+size(12);
+format %{ "LDR   $dst.lo,$mem\t! int -> long\n\t"
+"MOV    $dst.hi, 0\n\t"
+"AND   $dst,$dst,$mask" %}
+ins_encode %{
+__ ldr($dst$$Register, $mem$$Address);
+__ mov($dst$$Register->successor(), 0);
+__ andr($dst$$Register, $dst$$Register, $mask$$constant);
+%}
+ins_pipe(iload_mem);
+%}
+#endif
+#ifdef AARCH64
+// Load Integer with mask into a Long Register
+// FIXME: use signedRegI mask, remove tmp?
+instruct loadI2L_immI(iRegL dst, memoryI mem, immI mask, iRegI tmp) %{
+match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
+effect(TEMP dst, TEMP tmp);
+ins_cost(MEMORY_REF_COST + 3*DEFAULT_COST);
+format %{ "LDRSW    $mem,$dst\t! int & 31-bit mask -> long\n\t"
+"MOV_SLOW $tmp,$mask\n\t"
+"AND      $dst,$tmp,$dst" %}
+ins_encode %{
+__ ldrsw($dst$$Register, $mem$$Address);
+__ mov_slow($tmp$$Register, $mask$$constant);
+__ andr($dst$$Register, $dst$$Register, $tmp$$Register);
+%}
+ins_pipe(iload_mem);
+%}
+#else
+// Load Integer with a 31-bit mask into a Long Register
+// FIXME: use iRegI mask, remove tmp?
+instruct loadI2L_immU31(iRegL dst, memoryI mem, immU31 mask, iRegI tmp) %{
+match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
+effect(TEMP dst, TEMP tmp);
+ins_cost(MEMORY_REF_COST + 4*DEFAULT_COST);
+size(20);
+format %{ "LDR      $mem,$dst\t! int & 31-bit mask -> long\n\t"
+"MOV      $dst.hi, 0\n\t"
+"MOV_SLOW $tmp,$mask\n\t"
+"AND      $dst,$tmp,$dst" %}
+ins_encode %{
+__ ldr($dst$$Register, $mem$$Address);
+__ mov($dst$$Register->successor(), 0);
+__ mov_slow($tmp$$Register, $mask$$constant);
+__ andr($dst$$Register, $dst$$Register, $tmp$$Register);
+%}
+ins_pipe(iload_mem);
+%}
+#endif
+// Load Unsigned Integer into a Long Register
+instruct loadUI2L(iRegL dst, memoryI mem, immL_32bits mask) %{
+match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
+ins_cost(MEMORY_REF_COST);
+#ifdef AARCH64
+//size(4);
+format %{ "LDR_w $dst,$mem\t! uint -> long" %}
+ins_encode %{
+__ ldr_w($dst$$Register, $mem$$Address);
+%}
+#else
+size(8);
+format %{ "LDR   $dst.lo,$mem\t! uint -> long\n\t"
+"MOV   $dst.hi,0" %}
+ins_encode %{
+__ ldr($dst$$Register, $mem$$Address);
+__ mov($dst$$Register->successor(), 0);
+%}
+#endif
+ins_pipe(iload_mem);
+%}
+// Load Long
+#ifdef AARCH64
+// XXX This variant shouldn't be necessary if 6217251 is implemented
+instruct loadLoff(iRegLd dst, memoryScaledL mem, aimmX off, iRegP tmp) %{
+match(Set dst (LoadL (AddP mem off)));
+ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
+effect(TEMP tmp);
+size(4 * 2);
+format %{ "LDR    $dst,$mem+$off\t! long temp=$tmp" %}
+ins_encode %{
+Register base = reg_to_register_object($mem$$base);
+__ add($tmp$$Register, base, $off$$constant);
+Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
+__ ldr($dst$$Register, nmem);
+%}
+ins_pipe(iload_mem);
+%}
+#endif
+instruct loadL(iRegLd dst, memoryL mem ) %{
+#ifdef AARCH64
+// already atomic for Aarch64
+#else
+predicate(!((LoadLNode*)n)->require_atomic_access());
+#endif
+match(Set dst (LoadL mem));
+effect(TEMP dst);
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "ldr_64  $dst,$mem\t! long" %}
+ins_encode %{
+__ ldr_64($dst$$Register, $mem$$Address);
+%}
+ins_pipe(iload_mem);
+%}
+#ifndef AARCH64
+instruct loadL_2instr(iRegL dst, memorylong mem ) %{
+predicate(!((LoadLNode*)n)->require_atomic_access());
+match(Set dst (LoadL mem));
+ins_cost(MEMORY_REF_COST + DEFAULT_COST);
+size(8);
+format %{ "LDR    $dst.lo,$mem \t! long order of instrs reversed if $dst.lo == base($mem)\n\t"
+"LDR    $dst.hi,$mem+4 or $mem" %}
+ins_encode %{
+Address Amemlo = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
+Address Amemhi = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp + 4, relocInfo::none);
+if ($dst$$Register == reg_to_register_object($mem$$base)) {
+__ ldr($dst$$Register->successor(), Amemhi);
+__ ldr($dst$$Register, Amemlo);
+} else {
+__ ldr($dst$$Register, Amemlo);
+__ ldr($dst$$Register->successor(), Amemhi);
+}
+%}
+ins_pipe(iload_mem);
+%}
+instruct loadL_volatile(iRegL dst, indirect mem ) %{
+predicate(((LoadLNode*)n)->require_atomic_access());
+match(Set dst (LoadL mem));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "LDMIA    $dst,$mem\t! long" %}
+ins_encode %{
+// FIXME: why is ldmia considered atomic?  Should be ldrexd
+RegisterSet set($dst$$Register);
+set = set | reg_to_register_object($dst$$reg + 1);
+__ ldmia(reg_to_register_object($mem$$base), set);
+%}
+ins_pipe(iload_mem);
+%}
+instruct loadL_volatile_fp(iRegL dst, memoryD mem ) %{
+predicate(((LoadLNode*)n)->require_atomic_access());
+match(Set dst (LoadL mem));
+ins_cost(MEMORY_REF_COST);
+size(8);
+format %{ "FLDD      S14, $mem"
+"FMRRD    $dst, S14\t! long \n't" %}
+ins_encode %{
+__ fldd(S14, $mem$$Address);
+__ fmrrd($dst$$Register, $dst$$Register->successor(), S14);
+%}
+ins_pipe(iload_mem);
+%}
+instruct loadL_unaligned(iRegL dst, memorylong mem ) %{
+match(Set dst (LoadL_unaligned mem));
+ins_cost(MEMORY_REF_COST);
+size(8);
+format %{ "LDR    $dst.lo,$mem\t! long order of instrs reversed if $dst.lo == base($mem)\n\t"
+"LDR    $dst.hi,$mem+4" %}
+ins_encode %{
+Address Amemlo = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
+Address Amemhi = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp + 4, relocInfo::none);
+if ($dst$$Register == reg_to_register_object($mem$$base)) {
+__ ldr($dst$$Register->successor(), Amemhi);
+__ ldr($dst$$Register, Amemlo);
+} else {
+__ ldr($dst$$Register, Amemlo);
+__ ldr($dst$$Register->successor(), Amemhi);
+}
+%}
+ins_pipe(iload_mem);
+%}
+#endif // !AARCH64
+// Load Range
+instruct loadRange(iRegI dst, memoryI mem) %{
+match(Set dst (LoadRange mem));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "LDR_u32 $dst,$mem\t! range" %}
+ins_encode %{
+__ ldr_u32($dst$$Register, $mem$$Address);
+%}
+ins_pipe(iload_mem);
+%}
+// Load Pointer
+#ifdef AARCH64
+// XXX This variant shouldn't be necessary if 6217251 is implemented
+instruct loadPoff(iRegP dst, memoryScaledP mem, aimmX off, iRegP tmp) %{
+match(Set dst (LoadP (AddP mem off)));
+ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
+effect(TEMP tmp);
+size(4 * 2);
+format %{ "LDR    $dst,$mem+$off\t! ptr temp=$tmp" %}
+ins_encode %{
+Register base = reg_to_register_object($mem$$base);
+__ add($tmp$$Register, base, $off$$constant);
+Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
+__ ldr($dst$$Register, nmem);
+%}
+ins_pipe(iload_mem);
+%}
+#endif
+instruct loadP(iRegP dst, memoryP mem) %{
+match(Set dst (LoadP mem));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "LDR   $dst,$mem\t! ptr" %}
+ins_encode %{
+__ ldr($dst$$Register, $mem$$Address);
+%}
+ins_pipe(iload_mem);
+%}
+#ifdef XXX
+// FIXME XXXX
+//instruct loadSP(iRegP dst, memoryP mem) %{
+instruct loadSP(SPRegP dst, memoryP mem, iRegP tmp) %{
+match(Set dst (LoadP mem));
+effect(TEMP tmp);
+ins_cost(MEMORY_REF_COST+1);
+size(8);
+format %{ "LDR   $tmp,$mem\t! ptr\n\t"
+"MOV   $dst,$tmp\t! ptr" %}
+ins_encode %{
+__ ldr($tmp$$Register, $mem$$Address);
+__ mov($dst$$Register, $tmp$$Register);
+%}
+ins_pipe(iload_mem);
+%}
+#endif
+#ifdef _LP64
+// Load Compressed Pointer
+// XXX This variant shouldn't be necessary if 6217251 is implemented
+instruct loadNoff(iRegN dst, memoryScaledI mem, aimmX off, iRegP tmp) %{
+match(Set dst (LoadN (AddP mem off)));
+ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
+effect(TEMP tmp);
+size(4 * 2);
+format %{ "ldr_u32 $dst,$mem+$off\t! compressed ptr temp=$tmp" %}
+ins_encode %{
+Register base = reg_to_register_object($mem$$base);
+__ add($tmp$$Register, base, $off$$constant);
+Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
+__ ldr_u32($dst$$Register, nmem);
+%}
+ins_pipe(iload_mem);
+%}
+instruct loadN(iRegN dst, memoryI mem) %{
+match(Set dst (LoadN mem));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "ldr_u32 $dst,$mem\t! compressed ptr" %}
+ins_encode %{
+__ ldr_u32($dst$$Register, $mem$$Address);
+%}
+ins_pipe(iload_mem);
+%}
+#endif
+// Load Klass Pointer
+instruct loadKlass(iRegP dst, memoryI mem) %{
+match(Set dst (LoadKlass mem));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "LDR   $dst,$mem\t! klass ptr" %}
+ins_encode %{
+__ ldr($dst$$Register, $mem$$Address);
+%}
+ins_pipe(iload_mem);
+%}
+#ifdef _LP64
+// Load narrow Klass Pointer
+instruct loadNKlass(iRegN dst, memoryI mem) %{
+match(Set dst (LoadNKlass mem));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "ldr_u32 $dst,$mem\t! compressed klass ptr" %}
+ins_encode %{
+__ ldr_u32($dst$$Register, $mem$$Address);
+%}
+ins_pipe(iload_mem);
+%}
+#endif
+#ifdef AARCH64
+// XXX This variant shouldn't be necessary if 6217251 is implemented
+instruct loadDoff(regD dst, memoryScaledD mem, aimmX off, iRegP tmp) %{
+match(Set dst (LoadD (AddP mem off)));
+ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
+effect(TEMP tmp);
+size(4 * 2);
+format %{ "ldr    $dst,$mem+$off\t! double temp=$tmp" %}
+ins_encode %{
+Register base = reg_to_register_object($mem$$base);
+__ add($tmp$$Register, base, $off$$constant);
+Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
+__ ldr_d($dst$$FloatRegister, nmem);
+%}
+ins_pipe(floadD_mem);
+%}
+#endif
+instruct loadD(regD dst, memoryD mem) %{
+match(Set dst (LoadD mem));
+ins_cost(MEMORY_REF_COST);
+size(4);
+// FIXME: needs to be atomic, but  ARMv7 A.R.M. guarantees
+// only LDREXD and STREXD are 64-bit single-copy atomic
+format %{ "FLDD   $dst,$mem" %}
+ins_encode %{
+__ ldr_double($dst$$FloatRegister, $mem$$Address);
+%}
+ins_pipe(floadD_mem);
+%}
+#ifndef AARCH64
+// Load Double - UNaligned
+instruct loadD_unaligned(regD_low dst, memoryF2 mem ) %{
+match(Set dst (LoadD_unaligned mem));
+ins_cost(MEMORY_REF_COST*2+DEFAULT_COST);
+size(8);
+format %{ "FLDS    $dst.lo,$mem\t! misaligned double\n"
+"\tFLDS    $dst.hi,$mem+4\t!" %}
+ins_encode %{
+Address Amemlo = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
+Address Amemhi = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp + 4, relocInfo::none);
+__ flds($dst$$FloatRegister, Amemlo);
+__ flds($dst$$FloatRegister->successor(), Amemhi);
+%}
+ins_pipe(iload_mem);
+%}
+#endif
+#ifdef AARCH64
+// XXX This variant shouldn't be necessary if 6217251 is implemented
+instruct loadFoff(regF dst, memoryScaledF mem, aimmX off, iRegP tmp) %{
+match(Set dst (LoadF (AddP mem off)));
+ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
+effect(TEMP tmp);
+size(4 * 2);
+format %{ "ldr    $dst,$mem+$off\t! float temp=$tmp" %}
+ins_encode %{
+Register base = reg_to_register_object($mem$$base);
+__ add($tmp$$Register, base, $off$$constant);
+Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
+__ ldr_s($dst$$FloatRegister, nmem);
+%}
+ins_pipe(floadF_mem);
+%}
+#endif
+instruct loadF(regF dst, memoryF mem) %{
+match(Set dst (LoadF mem));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "FLDS    $dst,$mem" %}
+ins_encode %{
+__ ldr_float($dst$$FloatRegister, $mem$$Address);
+%}
+ins_pipe(floadF_mem);
+%}
+#ifdef AARCH64
+instruct load_limmI(iRegI dst, limmI src) %{
+match(Set dst src);
+ins_cost(DEFAULT_COST + 1); // + 1 because MOV is preferred
+format %{ "ORR_w  $dst, ZR, $src\t! int"  %}
+ins_encode %{
+__ orr_w($dst$$Register, ZR, (uintx)$src$$constant);
+%}
+ins_pipe(ialu_imm);
+%}
+#endif
+// // Load Constant
+instruct loadConI( iRegI dst, immI src ) %{
+match(Set dst src);
+ins_cost(DEFAULT_COST * 3/2);
+format %{ "MOV_SLOW    $dst, $src" %}
+ins_encode %{
+__ mov_slow($dst$$Register, $src$$constant);
+%}
+ins_pipe(ialu_hi_lo_reg);
+%}
+instruct loadConIMov( iRegI dst, immIMov src ) %{
+match(Set dst src);
+size(4);
+format %{ "MOV    $dst, $src" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$constant);
+%}
+ins_pipe(ialu_imm);
+%}
+#ifndef AARCH64
+instruct loadConIMovn( iRegI dst, immIRotn src ) %{
+match(Set dst src);
+size(4);
+format %{ "MVN    $dst, ~$src" %}
+ins_encode %{
+__ mvn($dst$$Register, ~$src$$constant);
+%}
+ins_pipe(ialu_imm_n);
+%}
+#endif
+instruct loadConI16( iRegI dst, immI16 src ) %{
+match(Set dst src);
+size(4);
+#ifdef AARCH64
+format %{ "MOVZ_w  $dst, $src" %}
+#else
+format %{ "MOVW    $dst, $src" %}
+#endif
+ins_encode %{
+#ifdef AARCH64
+__ mov_w($dst$$Register, $src$$constant);
+#else
+__ movw($dst$$Register, $src$$constant);
+#endif
+%}
+ins_pipe(ialu_imm_n);
+%}
+instruct loadConP(iRegP dst, immP src) %{
+match(Set dst src);
+ins_cost(DEFAULT_COST * 3/2);
+format %{ "MOV_SLOW    $dst,$src\t!ptr" %}
+ins_encode %{
+relocInfo::relocType constant_reloc = _opnds[1]->constant_reloc();
+intptr_t val = $src$$constant;
+if (constant_reloc == relocInfo::oop_type) {
+__ mov_oop($dst$$Register, (jobject)val);
+} else if (constant_reloc == relocInfo::metadata_type) {
+__ mov_metadata($dst$$Register, (Metadata*)val);
+} else {
+__ mov_slow($dst$$Register, val);
+}
+%}
+ins_pipe(loadConP);
+%}
+instruct loadConP_poll(iRegP dst, immP_poll src) %{
+match(Set dst src);
+ins_cost(DEFAULT_COST);
+format %{ "MOV_SLOW    $dst,$src\t!ptr" %}
+ins_encode %{
+__ mov_slow($dst$$Register, $src$$constant);
+%}
+ins_pipe(loadConP_poll);
+%}
+#ifdef AARCH64
+instruct loadConP0(iRegP dst, immP0 src) %{
+match(Set dst src);
+ins_cost(DEFAULT_COST);
+format %{ "MOV    $dst,ZR\t!ptr" %}
+ins_encode %{
+__ mov($dst$$Register, ZR);
+%}
+ins_pipe(ialu_none);
+%}
+instruct loadConN(iRegN dst, immN src) %{
+match(Set dst src);
+ins_cost(DEFAULT_COST * 3/2);
+format %{ "SET    $dst,$src\t! compressed ptr" %}
+ins_encode %{
+Register dst = $dst$$Register;
+// FIXME: use $constanttablebase?
+__ set_narrow_oop(dst, (jobject)$src$$constant);
+%}
+ins_pipe(ialu_hi_lo_reg);
+%}
+instruct loadConN0(iRegN dst, immN0 src) %{
+match(Set dst src);
+ins_cost(DEFAULT_COST);
+format %{ "MOV    $dst,ZR\t! compressed ptr" %}
+ins_encode %{
+__ mov($dst$$Register, ZR);
+%}
+ins_pipe(ialu_none);
+%}
+instruct loadConNKlass(iRegN dst, immNKlass src) %{
+match(Set dst src);
+ins_cost(DEFAULT_COST * 3/2);
+format %{ "SET    $dst,$src\t! compressed klass ptr" %}
+ins_encode %{
+Register dst = $dst$$Register;
+// FIXME: use $constanttablebase?
+__ set_narrow_klass(dst, (Klass*)$src$$constant);
+%}
+ins_pipe(ialu_hi_lo_reg);
+%}
+instruct load_limmL(iRegL dst, limmL src) %{
+match(Set dst src);
+ins_cost(DEFAULT_COST);
+format %{ "ORR    $dst, ZR, $src\t! long"  %}
+ins_encode %{
+__ orr($dst$$Register, ZR, (uintx)$src$$constant);
+%}
+ins_pipe(loadConL);
+%}
+instruct load_immLMov(iRegL dst, immLMov src) %{
+match(Set dst src);
+ins_cost(DEFAULT_COST);
+format %{ "MOV    $dst, $src\t! long"  %}
+ins_encode %{
+__ mov($dst$$Register, $src$$constant);
+%}
+ins_pipe(loadConL);
+%}
+instruct loadConL(iRegL dst, immL src) %{
+match(Set dst src);
+ins_cost(DEFAULT_COST * 4); // worst case
+format %{ "mov_slow   $dst, $src\t! long"  %}
+ins_encode %{
+// FIXME: use $constanttablebase?
+__ mov_slow($dst$$Register, $src$$constant);
+%}
+ins_pipe(loadConL);
+%}
+#else
+instruct loadConL(iRegL dst, immL src) %{
+match(Set dst src);
+ins_cost(DEFAULT_COST * 4);
+format %{ "MOV_SLOW   $dst.lo, $src & 0x0FFFFFFFFL \t! long\n\t"
+"MOV_SLOW   $dst.hi, $src >> 32" %}
+ins_encode %{
+__ mov_slow(reg_to_register_object($dst$$reg), $src$$constant & 0x0FFFFFFFFL);
+__ mov_slow(reg_to_register_object($dst$$reg + 1), ((julong)($src$$constant)) >> 32);
+%}
+ins_pipe(loadConL);
+%}
+instruct loadConL16( iRegL dst, immL16 src ) %{
+match(Set dst src);
+ins_cost(DEFAULT_COST * 2);
+size(8);
+format %{ "MOVW    $dst.lo, $src \n\t"
+"MOVW    $dst.hi, 0 \n\t" %}
+ins_encode %{
+__ movw($dst$$Register, $src$$constant);
+__ movw($dst$$Register->successor(), 0);
+%}
+ins_pipe(ialu_imm);
+%}
+#endif
+instruct loadConF_imm8(regF dst, imm8F src) %{
+match(Set dst src);
+ins_cost(DEFAULT_COST);
+size(4);
+format %{ "FCONSTS      $dst, $src"%}
+ins_encode %{
+__ fconsts($dst$$FloatRegister, Assembler::float_num($src$$constant).imm8());
+%}
+ins_pipe(loadConFD); // FIXME
+%}
+#ifdef AARCH64
+instruct loadIConF(iRegI dst, immF src) %{
+match(Set dst src);
+ins_cost(DEFAULT_COST * 2);
+format %{ "MOV_SLOW  $dst, $src\t! loadIConF"  %}
+ins_encode %{
+// FIXME revisit once 6961697 is in
+union {
+jfloat f;
+int i;
+} v;
+v.f = $src$$constant;
+__ mov_slow($dst$$Register, v.i);
+%}
+ins_pipe(ialu_imm);
+%}
+#endif
+instruct loadConF(regF dst, immF src, iRegI tmp) %{
+match(Set dst src);
+ins_cost(DEFAULT_COST * 2);
+effect(TEMP tmp);
+size(3*4);
+format %{ "MOV_SLOW  $tmp, $src\n\t"
+"FMSR      $dst, $tmp"%}
+ins_encode %{
+// FIXME revisit once 6961697 is in
+union {
+jfloat f;
+int i;
+} v;
+v.f = $src$$constant;
+__ mov_slow($tmp$$Register, v.i);
+__ fmsr($dst$$FloatRegister, $tmp$$Register);
+%}
+ins_pipe(loadConFD); // FIXME
+%}
+instruct loadConD_imm8(regD dst, imm8D src) %{
+match(Set dst src);
+ins_cost(DEFAULT_COST);
+size(4);
+format %{ "FCONSTD      $dst, $src"%}
+ins_encode %{
+__ fconstd($dst$$FloatRegister, Assembler::double_num($src$$constant).imm8());
+%}
+ins_pipe(loadConFD); // FIXME
+%}
+instruct loadConD(regD dst, immD src, iRegP tmp) %{
+match(Set dst src);
+effect(TEMP tmp);
+ins_cost(MEMORY_REF_COST);
+format %{ "FLDD  $dst, [$constanttablebase + $constantoffset]\t! load from constant table: double=$src" %}
+ins_encode %{
+Register r = $constanttablebase;
+int offset  = $constantoffset($src);
+if (!is_memoryD(offset)) {                // can't use a predicate
+// in load constant instructs
+__ add_slow($tmp$$Register, r, offset);
+r = $tmp$$Register;
+offset = 0;
+}
+__ ldr_double($dst$$FloatRegister, Address(r, offset));
+%}
+ins_pipe(loadConFD);
+%}
+// Prefetch instructions.
+// Must be safe to execute with invalid address (cannot fault).
+instruct prefetchAlloc_mp( memoryP mem ) %{
+predicate(os::is_MP());
+match( PrefetchAllocation mem );
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "PLDW $mem\t! Prefetch allocation" %}
+ins_encode %{
+#ifdef AARCH64
+__ prfm(pstl1keep, $mem$$Address);
+#else
+__ pldw($mem$$Address);
+#endif
+%}
+ins_pipe(iload_mem);
+%}
+instruct prefetchAlloc_sp( memoryP mem ) %{
+predicate(!os::is_MP());
+match( PrefetchAllocation mem );
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "PLD $mem\t! Prefetch allocation" %}
+ins_encode %{
+#ifdef AARCH64
+__ prfm(pstl1keep, $mem$$Address);
+#else
+__ pld($mem$$Address);
+#endif
+%}
+ins_pipe(iload_mem);
+%}
+//----------Store Instructions-------------------------------------------------
+// Store Byte
+instruct storeB(memoryB mem, store_RegI src) %{
+match(Set mem (StoreB mem src));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "STRB    $src,$mem\t! byte" %}
+ins_encode %{
+__ strb($src$$Register, $mem$$Address);
+%}
+ins_pipe(istore_mem_reg);
+%}
+instruct storeCM(memoryB mem, store_RegI src) %{
+match(Set mem (StoreCM mem src));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "STRB    $src,$mem\t! CMS card-mark byte" %}
+ins_encode %{
+__ strb($src$$Register, $mem$$Address);
+%}
+ins_pipe(istore_mem_reg);
+%}
+// Store Char/Short
+#ifdef AARCH64
+// XXX This variant shouldn't be necessary if 6217251 is implemented
+instruct storeCoff(store_RegI src, memoryScaledS mem, aimmX off, iRegP tmp) %{
+match(Set mem (StoreC (AddP mem off) src));
+ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
+effect(TEMP tmp);
+size(4 * 2);
+format %{ "STRH    $src,$mem+$off\t! short temp=$tmp" %}
+ins_encode %{
+Register base = reg_to_register_object($mem$$base);
+__ add($tmp$$Register, base, $off$$constant);
+Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
+__ strh($src$$Register, nmem);
+%}
+ins_pipe(istore_mem_reg);
+%}
+#endif
+instruct storeC(memoryS mem, store_RegI src) %{
+match(Set mem (StoreC mem src));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "STRH    $src,$mem\t! short" %}
+ins_encode %{
+__ strh($src$$Register, $mem$$Address);
+%}
+ins_pipe(istore_mem_reg);
+%}
+// Store Integer
+#ifdef AARCH64
+// XXX This variant shouldn't be necessary if 6217251 is implemented
+instruct storeIoff(store_RegI src, memoryScaledI mem, aimmX off, iRegP tmp) %{
+match(Set mem (StoreI (AddP mem off) src));
+ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
+effect(TEMP tmp);
+size(4 * 2);
+format %{ "str_32 $src,$mem+$off\t! int temp=$tmp" %}
+ins_encode %{
+Register base = reg_to_register_object($mem$$base);
+__ add($tmp$$Register, base, $off$$constant);
+Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
+__ str_32($src$$Register, nmem);
+%}
+ins_pipe(istore_mem_reg);
+%}
+#endif
+instruct storeI(memoryI mem, store_RegI src) %{
+match(Set mem (StoreI mem src));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "str_32 $src,$mem" %}
+ins_encode %{
+__ str_32($src$$Register, $mem$$Address);
+%}
+ins_pipe(istore_mem_reg);
+%}
+// Store Long
+#ifdef AARCH64
+// XXX This variant shouldn't be necessary if 6217251 is implemented
+instruct storeLoff(store_RegLd src, memoryScaledL mem, aimmX off, iRegP tmp) %{
+match(Set mem (StoreL (AddP mem off) src));
+ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
+effect(TEMP tmp);
+size(4 * 2);
+format %{ "str_64 $src,$mem+$off\t! long temp=$tmp" %}
+ins_encode %{
+Register base = reg_to_register_object($mem$$base);
+__ add($tmp$$Register, base, $off$$constant);
+Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
+__ str_64($src$$Register, nmem);
+%}
+ins_pipe(istore_mem_reg);
+%}
+#endif
+instruct storeL(memoryL mem, store_RegLd src) %{
+#ifdef AARCH64
+// already atomic for Aarch64
+#else
+predicate(!((StoreLNode*)n)->require_atomic_access());
+#endif
+match(Set mem (StoreL mem src));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "str_64  $src,$mem\t! long\n\t" %}
+ins_encode %{
+__ str_64($src$$Register, $mem$$Address);
+%}
+ins_pipe(istore_mem_reg);
+%}
+#ifndef AARCH64
+instruct storeL_2instr(memorylong mem, iRegL src) %{
+predicate(!((StoreLNode*)n)->require_atomic_access());
+match(Set mem (StoreL mem src));
+ins_cost(MEMORY_REF_COST + DEFAULT_COST);
+size(8);
+format %{ "STR    $src.lo,$mem\t! long\n\t"
+"STR    $src.hi,$mem+4" %}
+ins_encode %{
+Address Amemlo = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
+Address Amemhi = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp + 4, relocInfo::none);
+__ str($src$$Register, Amemlo);
+__ str($src$$Register->successor(), Amemhi);
+%}
+ins_pipe(istore_mem_reg);
+%}
+instruct storeL_volatile(indirect mem, iRegL src) %{
+predicate(((StoreLNode*)n)->require_atomic_access());
+match(Set mem (StoreL mem src));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "STMIA    $src,$mem\t! long" %}
+ins_encode %{
+// FIXME: why is stmia considered atomic?  Should be strexd
+RegisterSet set($src$$Register);
+set = set | reg_to_register_object($src$$reg + 1);
+__ stmia(reg_to_register_object($mem$$base), set);
+%}
+ins_pipe(istore_mem_reg);
+%}
+#endif // !AARCH64
+#ifndef AARCH64
+instruct storeL_volatile_fp(memoryD mem, iRegL src) %{
+predicate(((StoreLNode*)n)->require_atomic_access());
+match(Set mem (StoreL mem src));
+ins_cost(MEMORY_REF_COST);
+size(8);
+format %{ "FMDRR    S14, $src\t! long \n\t"
+"FSTD     S14, $mem" %}
+ins_encode %{
+__ fmdrr(S14, $src$$Register, $src$$Register->successor());
+__ fstd(S14, $mem$$Address);
+%}
+ins_pipe(istore_mem_reg);
+%}
+#endif
+#ifdef XXX
+// Move SP Pointer
+//instruct movSP(sp_ptr_RegP dst, SPRegP src) %{
+//instruct movSP(iRegP dst, SPRegP src) %{
+instruct movSP(store_ptr_RegP dst, SPRegP src) %{
+match(Set dst src);
+//predicate(!_kids[1]->_leaf->is_Proj() || _kids[1]->_leaf->as_Proj()->_con == TypeFunc::FramePtr);
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "MOV    $dst,$src\t! SP ptr\n\t" %}
+ins_encode %{
+assert(false, "XXX1 got here");
+__ mov($dst$$Register, SP);
+__ mov($dst$$Register, $src$$Register);
+%}
+ins_pipe(ialu_reg);
+%}
+#endif
+#ifdef AARCH64
+// FIXME
+// Store SP Pointer
+instruct storeSP(memoryP mem, SPRegP src, iRegP tmp) %{
+match(Set mem (StoreP mem src));
+predicate(_kids[1]->_leaf->is_Proj() && _kids[1]->_leaf->as_Proj()->_con == TypeFunc::FramePtr);
+// Multiple StoreP rules, different only in register mask.
+// Matcher makes the last always valid.  The others will
+// only be valid if they cost less than the last valid
+// rule.  So cost(rule1) < cost(rule2) < cost(last)
+// Unlike immediates, register constraints are not checked
+// at match time.
+ins_cost(MEMORY_REF_COST+DEFAULT_COST+4);
+effect(TEMP tmp);
+size(8);
+format %{ "MOV    $tmp,$src\t! SP ptr\n\t"
+"STR    $tmp,$mem\t! SP ptr" %}
+ins_encode %{
+assert($src$$Register == SP, "SP expected");
+__ mov($tmp$$Register, $src$$Register);
+__ str($tmp$$Register, $mem$$Address);
+%}
+ins_pipe(istore_mem_spORreg); // FIXME
+%}
+#endif // AARCH64
+// Store Pointer
+#ifdef AARCH64
+// XXX This variant shouldn't be necessary if 6217251 is implemented
+instruct storePoff(store_ptr_RegP src, memoryScaledP mem, aimmX off, iRegP tmp) %{
+predicate(!_kids[1]->_leaf->is_Proj() || _kids[1]->_leaf->as_Proj()->_con != TypeFunc::FramePtr);
+match(Set mem (StoreP (AddP mem off) src));
+ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
+effect(TEMP tmp);
+size(4 * 2);
+format %{ "STR    $src,$mem+$off\t! ptr temp=$tmp" %}
+ins_encode %{
+Register base = reg_to_register_object($mem$$base);
+__ add($tmp$$Register, base, $off$$constant);
+Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
+__ str($src$$Register, nmem);
+%}
+ins_pipe(istore_mem_reg);
+%}
+#endif
+instruct storeP(memoryP mem, store_ptr_RegP src) %{
+match(Set mem (StoreP mem src));
+#ifdef AARCH64
+predicate(!_kids[1]->_leaf->is_Proj() || _kids[1]->_leaf->as_Proj()->_con != TypeFunc::FramePtr);
+#endif
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "STR    $src,$mem\t! ptr" %}
+ins_encode %{
+__ str($src$$Register, $mem$$Address);
+%}
+ins_pipe(istore_mem_spORreg);
+%}
+#ifdef AARCH64
+// Store NULL Pointer
+instruct storeP0(memoryP mem, immP0 src) %{
+match(Set mem (StoreP mem src));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "STR    ZR,$mem\t! ptr" %}
+ins_encode %{
+__ str(ZR, $mem$$Address);
+%}
+ins_pipe(istore_mem_spORreg);
+%}
+#endif // AARCH64
+#ifdef _LP64
+// Store Compressed Pointer
+#ifdef AARCH64
+// XXX This variant shouldn't be necessary if 6217251 is implemented
+instruct storeNoff(store_RegN src, memoryScaledI mem, aimmX off, iRegP tmp) %{
+match(Set mem (StoreN (AddP mem off) src));
+ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
+effect(TEMP tmp);
+size(4 * 2);
+format %{ "str_32 $src,$mem+$off\t! compressed ptr temp=$tmp" %}
+ins_encode %{
+Register base = reg_to_register_object($mem$$base);
+__ add($tmp$$Register, base, $off$$constant);
+Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
+__ str_32($src$$Register, nmem);
+%}
+ins_pipe(istore_mem_reg);
+%}
+#endif
+instruct storeN(memoryI mem, store_RegN src) %{
+match(Set mem (StoreN mem src));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "str_32 $src,$mem\t! compressed ptr" %}
+ins_encode %{
+__ str_32($src$$Register, $mem$$Address);
+%}
+ins_pipe(istore_mem_reg);
+%}
+#ifdef AARCH64
+// Store NULL Pointer
+instruct storeN0(memoryI mem, immN0 src) %{
+match(Set mem (StoreN mem src));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "str_32 ZR,$mem\t! compressed ptr" %}
+ins_encode %{
+__ str_32(ZR, $mem$$Address);
+%}
+ins_pipe(istore_mem_reg);
+%}
+#endif
+// Store Compressed Klass Pointer
+instruct storeNKlass(memoryI mem, store_RegN src) %{
+match(Set mem (StoreNKlass mem src));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "str_32 $src,$mem\t! compressed klass ptr" %}
+ins_encode %{
+__ str_32($src$$Register, $mem$$Address);
+%}
+ins_pipe(istore_mem_reg);
+%}
+#endif
+// Store Double
+#ifdef AARCH64
+// XXX This variant shouldn't be necessary if 6217251 is implemented
+instruct storeDoff(regD src, memoryScaledD mem, aimmX off, iRegP tmp) %{
+match(Set mem (StoreD (AddP mem off) src));
+ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
+effect(TEMP tmp);
+size(4 * 2);
+format %{ "STR    $src,$mem+$off\t! double temp=$tmp" %}
+ins_encode %{
+Register base = reg_to_register_object($mem$$base);
+__ add($tmp$$Register, base, $off$$constant);
+Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
+__ str_d($src$$FloatRegister, nmem);
+%}
+ins_pipe(fstoreD_mem_reg);
+%}
+#endif
+instruct storeD(memoryD mem, regD src) %{
+match(Set mem (StoreD mem src));
+ins_cost(MEMORY_REF_COST);
+size(4);
+// FIXME: needs to be atomic, but  ARMv7 A.R.M. guarantees
+// only LDREXD and STREXD are 64-bit single-copy atomic
+format %{ "FSTD   $src,$mem" %}
+ins_encode %{
+__ str_double($src$$FloatRegister, $mem$$Address);
+%}
+ins_pipe(fstoreD_mem_reg);
+%}
+#ifdef AARCH64
+instruct movI2F(regF dst, iRegI src) %{
+match(Set dst src);
+size(4);
+format %{ "FMOV_sw $dst,$src\t! movI2F" %}
+ins_encode %{
+__ fmov_sw($dst$$FloatRegister, $src$$Register);
+%}
+ins_pipe(ialu_reg); // FIXME
+%}
+instruct movF2I(iRegI dst, regF src) %{
+match(Set dst src);
+size(4);
+format %{ "FMOV_ws $dst,$src\t! movF2I" %}
+ins_encode %{
+__ fmov_ws($dst$$Register, $src$$FloatRegister);
+%}
+ins_pipe(ialu_reg); // FIXME
+%}
+#endif
+// Store Float
+#ifdef AARCH64
+// XXX This variant shouldn't be necessary if 6217251 is implemented
+instruct storeFoff(regF src, memoryScaledF mem, aimmX off, iRegP tmp) %{
+match(Set mem (StoreF (AddP mem off) src));
+ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
+effect(TEMP tmp);
+size(4 * 2);
+format %{ "str_s  $src,$mem+$off\t! float temp=$tmp" %}
+ins_encode %{
+Register base = reg_to_register_object($mem$$base);
+__ add($tmp$$Register, base, $off$$constant);
+Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
+__ str_s($src$$FloatRegister, nmem);
+%}
+ins_pipe(fstoreF_mem_reg);
+%}
+#endif
+instruct storeF( memoryF mem, regF src) %{
+match(Set mem (StoreF mem src));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "FSTS    $src,$mem" %}
+ins_encode %{
+__ str_float($src$$FloatRegister, $mem$$Address);
+%}
+ins_pipe(fstoreF_mem_reg);
+%}
+#ifdef AARCH64
+// Convert oop pointer into compressed form
+instruct encodeHeapOop(iRegN dst, iRegP src, flagsReg ccr) %{
+predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
+match(Set dst (EncodeP src));
+effect(KILL ccr);
+format %{ "encode_heap_oop $dst, $src" %}
+ins_encode %{
+__ encode_heap_oop($dst$$Register, $src$$Register);
+%}
+ins_pipe(ialu_reg);
+%}
+instruct encodeHeapOop_not_null(iRegN dst, iRegP src) %{
+predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
+match(Set dst (EncodeP src));
+format %{ "encode_heap_oop_not_null $dst, $src" %}
+ins_encode %{
+__ encode_heap_oop_not_null($dst$$Register, $src$$Register);
+%}
+ins_pipe(ialu_reg);
+%}
+instruct decodeHeapOop(iRegP dst, iRegN src, flagsReg ccr) %{
+predicate(n->bottom_type()->is_oopptr()->ptr() != TypePtr::NotNull &&
+n->bottom_type()->is_oopptr()->ptr() != TypePtr::Constant);
+match(Set dst (DecodeN src));
+effect(KILL ccr);
+format %{ "decode_heap_oop $dst, $src" %}
+ins_encode %{
+__ decode_heap_oop($dst$$Register, $src$$Register);
+%}
+ins_pipe(ialu_reg);
+%}
+instruct decodeHeapOop_not_null(iRegP dst, iRegN src) %{
+predicate(n->bottom_type()->is_oopptr()->ptr() == TypePtr::NotNull ||
+n->bottom_type()->is_oopptr()->ptr() == TypePtr::Constant);
+match(Set dst (DecodeN src));
+format %{ "decode_heap_oop_not_null $dst, $src" %}
+ins_encode %{
+__ decode_heap_oop_not_null($dst$$Register, $src$$Register);
+%}
+ins_pipe(ialu_reg);
+%}
+instruct encodeKlass_not_null(iRegN dst, iRegP src) %{
+match(Set dst (EncodePKlass src));
+format %{ "encode_klass_not_null $dst, $src" %}
+ins_encode %{
+__ encode_klass_not_null($dst$$Register, $src$$Register);
+%}
+ins_pipe(ialu_reg);
+%}
+instruct decodeKlass_not_null(iRegP dst, iRegN src) %{
+match(Set dst (DecodeNKlass src));
+format %{ "decode_klass_not_null $dst, $src" %}
+ins_encode %{
+__ decode_klass_not_null($dst$$Register, $src$$Register);
+%}
+ins_pipe(ialu_reg);
+%}
+#endif // AARCH64
+//----------MemBar Instructions-----------------------------------------------
+// Memory barrier flavors
+// TODO: take advantage of Aarch64 load-acquire, store-release, etc
+// pattern-match out unnecessary membars
+instruct membar_storestore() %{
+match(MemBarStoreStore);
+ins_cost(4*MEMORY_REF_COST);
+size(4);
+format %{ "MEMBAR-storestore" %}
+ins_encode %{
+__ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore), noreg);
+%}
+ins_pipe(long_memory_op);
+%}
+instruct membar_acquire() %{
+match(MemBarAcquire);
+match(LoadFence);
+ins_cost(4*MEMORY_REF_COST);
+size(4);
+format %{ "MEMBAR-acquire" %}
+ins_encode %{
+__ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::LoadLoad | MacroAssembler::LoadStore), noreg);
+%}
+ins_pipe(long_memory_op);
+%}
+instruct membar_acquire_lock() %{
+match(MemBarAcquireLock);
+ins_cost(0);
+size(0);
+format %{ "!MEMBAR-acquire (CAS in prior FastLock so empty encoding)" %}
+ins_encode( );
+ins_pipe(empty);
+%}
+instruct membar_release() %{
+match(MemBarRelease);
+match(StoreFence);
+ins_cost(4*MEMORY_REF_COST);
+size(4);
+format %{ "MEMBAR-release" %}
+ins_encode %{
+__ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore | MacroAssembler::LoadStore), noreg);
+%}
+ins_pipe(long_memory_op);
+%}
+instruct membar_release_lock() %{
+match(MemBarReleaseLock);
+ins_cost(0);
+size(0);
+format %{ "!MEMBAR-release (CAS in succeeding FastUnlock so empty encoding)" %}
+ins_encode( );
+ins_pipe(empty);
+%}
+instruct membar_volatile() %{
+match(MemBarVolatile);
+ins_cost(4*MEMORY_REF_COST);
+size(4);
+format %{ "MEMBAR-volatile" %}
+ins_encode %{
+__ membar(MacroAssembler::StoreLoad, noreg);
+%}
+ins_pipe(long_memory_op);
+%}
+instruct unnecessary_membar_volatile() %{
+match(MemBarVolatile);
+predicate(Matcher::post_store_load_barrier(n));
+ins_cost(0);
+size(0);
+format %{ "!MEMBAR-volatile (unnecessary so empty encoding)" %}
+ins_encode( );
+ins_pipe(empty);
+%}
+//----------Register Move Instructions-----------------------------------------
+// instruct roundDouble_nop(regD dst) %{
+//   match(Set dst (RoundDouble dst));
+//   ins_pipe(empty);
+// %}
+// instruct roundFloat_nop(regF dst) %{
+//   match(Set dst (RoundFloat dst));
+//   ins_pipe(empty);
+// %}
+#ifdef AARCH64
+// 0 constant in register
+instruct zrImmI0(ZRRegI dst, immI0 imm) %{
+match(Set dst imm);
+size(0);
+ins_cost(0);
+format %{ "! ZR (int 0)" %}
+ins_encode( /*empty encoding*/ );
+ins_pipe(ialu_none);
+%}
+// 0 constant in register
+instruct zrImmL0(ZRRegL dst, immL0 imm) %{
+match(Set dst imm);
+size(0);
+ins_cost(0);
+format %{ "! ZR (long 0)" %}
+ins_encode( /*empty encoding*/ );
+ins_pipe(ialu_none);
+%}
+#ifdef XXX
+// 0 constant in register
+instruct zrImmN0(ZRRegN dst, immN0 imm) %{
+match(Set dst imm);
+size(0);
+ins_cost(0);
+format %{ "! ZR (compressed pointer NULL)" %}
+ins_encode( /*empty encoding*/ );
+ins_pipe(ialu_none);
+%}
+// 0 constant in register
+instruct zrImmP0(ZRRegP dst, immP0 imm) %{
+match(Set dst imm);
+size(0);
+ins_cost(0);
+format %{ "! ZR (NULL)" %}
+ins_encode( /*empty encoding*/ );
+ins_pipe(ialu_none);
+%}
+#endif
+#endif // AARCH64
+// Cast Index to Pointer for unsafe natives
+instruct castX2P(iRegX src, iRegP dst) %{
+match(Set dst (CastX2P src));
+format %{ "MOV    $dst,$src\t! IntX->Ptr if $dst != $src" %}
+ins_encode %{
+if ($dst$$Register !=  $src$$Register) {
+__ mov($dst$$Register, $src$$Register);
+}
+%}
+ins_pipe(ialu_reg);
+%}
+// Cast Pointer to Index for unsafe natives
+instruct castP2X(iRegP src, iRegX dst) %{
+match(Set dst (CastP2X src));
+format %{ "MOV    $dst,$src\t! Ptr->IntX if $dst != $src" %}
+ins_encode %{
+if ($dst$$Register !=  $src$$Register) {
+__ mov($dst$$Register, $src$$Register);
+}
+%}
+ins_pipe(ialu_reg);
+%}
+#ifndef AARCH64
+//----------Conditional Move---------------------------------------------------
+// Conditional move
+instruct cmovIP_reg(cmpOpP cmp, flagsRegP pcc, iRegI dst, iRegI src) %{
+match(Set dst (CMoveI (Binary cmp pcc) (Binary dst src)));
+ins_cost(150);
+size(4);
+format %{ "MOV$cmp  $dst,$src\t! int" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+#endif
+#ifdef AARCH64
+instruct cmovI_reg3(cmpOp cmp, flagsReg icc, iRegI dst, iRegI src1, iRegI src2) %{
+match(Set dst (CMoveI (Binary cmp icc) (Binary src2 src1)));
+ins_cost(150);
+size(4);
+format %{ "CSEL $dst,$src1,$src2,$cmp\t! int" %}
+ins_encode %{
+__ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+instruct cmovL_reg3(cmpOp cmp, flagsReg icc, iRegL dst, iRegL src1, iRegL src2) %{
+match(Set dst (CMoveL (Binary cmp icc) (Binary src2 src1)));
+ins_cost(150);
+size(4);
+format %{ "CSEL $dst,$src1,$src2,$cmp\t! long" %}
+ins_encode %{
+__ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+instruct cmovP_reg3(cmpOp cmp, flagsReg icc, iRegP dst, iRegP src1, iRegP src2) %{
+match(Set dst (CMoveP (Binary cmp icc) (Binary src2 src1)));
+ins_cost(150);
+size(4);
+format %{ "CSEL $dst,$src1,$src2,$cmp\t! ptr" %}
+ins_encode %{
+__ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+instruct cmovN_reg3(cmpOp cmp, flagsReg icc, iRegN dst, iRegN src1, iRegN src2) %{
+match(Set dst (CMoveN (Binary cmp icc) (Binary src2 src1)));
+ins_cost(150);
+size(4);
+format %{ "CSEL $dst,$src1,$src2,$cmp\t! compressed ptr" %}
+ins_encode %{
+__ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+instruct cmovIP_reg3(cmpOpP cmp, flagsRegP icc, iRegI dst, iRegI src1, iRegI src2) %{
+match(Set dst (CMoveI (Binary cmp icc) (Binary src2 src1)));
+ins_cost(150);
+size(4);
+format %{ "CSEL $dst,$src1,$src2,$cmp\t! int" %}
+ins_encode %{
+__ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+instruct cmovLP_reg3(cmpOpP cmp, flagsRegP icc, iRegL dst, iRegL src1, iRegL src2) %{
+match(Set dst (CMoveL (Binary cmp icc) (Binary src2 src1)));
+ins_cost(150);
+size(4);
+format %{ "CSEL $dst,$src1,$src2,$cmp\t! long" %}
+ins_encode %{
+__ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+instruct cmovPP_reg3(cmpOpP cmp, flagsRegP icc, iRegP dst, iRegP src1, iRegP src2) %{
+match(Set dst (CMoveP (Binary cmp icc) (Binary src2 src1)));
+ins_cost(150);
+size(4);
+format %{ "CSEL $dst,$src1,$src2,$cmp\t! ptr" %}
+ins_encode %{
+__ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+instruct cmovNP_reg3(cmpOpP cmp, flagsRegP icc, iRegN dst, iRegN src1, iRegN src2) %{
+match(Set dst (CMoveN (Binary cmp icc) (Binary src2 src1)));
+ins_cost(150);
+size(4);
+format %{ "CSEL $dst,$src1,$src2,$cmp\t! compressed ptr" %}
+ins_encode %{
+__ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+instruct cmovIU_reg3(cmpOpU cmp, flagsRegU icc, iRegI dst, iRegI src1, iRegI src2) %{
+match(Set dst (CMoveI (Binary cmp icc) (Binary src2 src1)));
+ins_cost(150);
+size(4);
+format %{ "CSEL $dst,$src1,$src2,$cmp\t! int" %}
+ins_encode %{
+__ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+instruct cmovLU_reg3(cmpOpU cmp, flagsRegU icc, iRegL dst, iRegL src1, iRegL src2) %{
+match(Set dst (CMoveL (Binary cmp icc) (Binary src2 src1)));
+ins_cost(150);
+size(4);
+format %{ "CSEL $dst,$src1,$src2,$cmp\t! long" %}
+ins_encode %{
+__ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+instruct cmovPU_reg3(cmpOpU cmp, flagsRegU icc, iRegP dst, iRegP src1, iRegP src2) %{
+match(Set dst (CMoveP (Binary cmp icc) (Binary src2 src1)));
+ins_cost(150);
+size(4);
+format %{ "CSEL $dst,$src1,$src2,$cmp\t! ptr" %}
+ins_encode %{
+__ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+instruct cmovNU_reg3(cmpOpU cmp, flagsRegU icc, iRegN dst, iRegN src1, iRegN src2) %{
+match(Set dst (CMoveN (Binary cmp icc) (Binary src2 src1)));
+ins_cost(150);
+size(4);
+format %{ "CSEL $dst,$src1,$src2,$cmp\t! compressed ptr" %}
+ins_encode %{
+__ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+instruct cmovIZ_reg3(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegI dst, iRegI src1, iRegI src2) %{
+match(Set dst (CMoveI (Binary cmp icc) (Binary src2 src1)));
+ins_cost(150);
+size(4);
+format %{ "CSEL $dst,$src1,$src2,$cmp\t! int" %}
+ins_encode %{
+__ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+instruct cmovLZ_reg3(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegL dst, iRegL src1, iRegL src2) %{
+match(Set dst (CMoveL (Binary cmp icc) (Binary src2 src1)));
+ins_cost(150);
+size(4);
+format %{ "CSEL $dst,$src1,$src2,$cmp\t! long" %}
+ins_encode %{
+__ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+instruct cmovPZ_reg3(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegP dst, iRegP src1, iRegP src2) %{
+match(Set dst (CMoveP (Binary cmp icc) (Binary src2 src1)));
+ins_cost(150);
+size(4);
+format %{ "CSEL $dst,$src1,$src2,$cmp\t! ptr" %}
+ins_encode %{
+__ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+instruct cmovNZ_reg3(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegN dst, iRegN src1, iRegN src2) %{
+match(Set dst (CMoveN (Binary cmp icc) (Binary src2 src1)));
+ins_cost(150);
+size(4);
+format %{ "CSEL $dst,$src1,$src2,$cmp\t! compressed ptr" %}
+ins_encode %{
+__ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+#endif // AARCH64
+#ifndef AARCH64
+instruct cmovIP_immMov(cmpOpP cmp, flagsRegP pcc, iRegI dst, immIMov src) %{
+match(Set dst (CMoveI (Binary cmp pcc) (Binary dst src)));
+ins_cost(140);
+size(4);
+format %{ "MOV$cmp  $dst,$src" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_imm);
+%}
+instruct cmovIP_imm16(cmpOpP cmp, flagsRegP pcc, iRegI dst, immI16 src) %{
+match(Set dst (CMoveI (Binary cmp pcc) (Binary dst src)));
+ins_cost(140);
+size(4);
+format %{ "MOVw$cmp  $dst,$src" %}
+ins_encode %{
+__ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_imm);
+%}
+#endif
+instruct cmovI_reg(cmpOp cmp, flagsReg icc, iRegI dst, iRegI src) %{
+match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
+ins_cost(150);
+size(4);
+format %{ "MOV$cmp  $dst,$src" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+#ifdef AARCH64
+instruct cmovL_reg(cmpOp cmp, flagsReg icc, iRegL dst, iRegL src) %{
+match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
+ins_cost(150);
+size(4);
+format %{ "MOV$cmp  $dst,$src\t! long" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+#endif
+#ifndef AARCH64
+instruct cmovI_immMov(cmpOp cmp, flagsReg icc, iRegI dst, immIMov src) %{
+match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
+ins_cost(140);
+size(4);
+format %{ "MOV$cmp  $dst,$src" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_imm);
+%}
+instruct cmovII_imm16(cmpOp cmp, flagsReg icc, iRegI dst, immI16 src) %{
+match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
+ins_cost(140);
+size(4);
+format %{ "MOVw$cmp  $dst,$src" %}
+ins_encode %{
+__ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_imm);
+%}
+#endif
+instruct cmovII_reg_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegI dst, iRegI src) %{
+match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
+_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
+_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
+_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
+ins_cost(150);
+size(4);
+format %{ "MOV$cmp  $dst,$src" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+#ifndef AARCH64
+instruct cmovII_immMov_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegI dst, immIMov src) %{
+match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
+_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
+_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
+_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
+ins_cost(140);
+size(4);
+format %{ "MOV$cmp  $dst,$src" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_imm);
+%}
+instruct cmovII_imm16_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegI dst, immI16 src) %{
+match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
+_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
+_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
+_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
+ins_cost(140);
+size(4);
+format %{ "MOVW$cmp  $dst,$src" %}
+ins_encode %{
+__ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_imm);
+%}
+#endif
+instruct cmovIIu_reg(cmpOpU cmp, flagsRegU icc, iRegI dst, iRegI src) %{
+match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
+ins_cost(150);
+size(4);
+format %{ "MOV$cmp  $dst,$src" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+#ifndef AARCH64
+instruct cmovIIu_immMov(cmpOpU cmp, flagsRegU icc, iRegI dst, immIMov src) %{
+match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
+ins_cost(140);
+size(4);
+format %{ "MOV$cmp  $dst,$src" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_imm);
+%}
+instruct cmovIIu_imm16(cmpOpU cmp, flagsRegU icc, iRegI dst, immI16 src) %{
+match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
+ins_cost(140);
+size(4);
+format %{ "MOVW$cmp  $dst,$src" %}
+ins_encode %{
+__ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_imm);
+%}
+#endif
+// Conditional move
+instruct cmovPP_reg(cmpOpP cmp, flagsRegP pcc, iRegP dst, iRegP src) %{
+match(Set dst (CMoveP (Binary cmp pcc) (Binary dst src)));
+ins_cost(150);
+size(4);
+format %{ "MOV$cmp  $dst,$src" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+instruct cmovPP_imm(cmpOpP cmp, flagsRegP pcc, iRegP dst, immP0 src) %{
+match(Set dst (CMoveP (Binary cmp pcc) (Binary dst src)));
+ins_cost(140);
+size(4);
+#ifdef AARCH64
+format %{ "MOV$cmp  $dst,ZR" %}
+#else
+format %{ "MOV$cmp  $dst,$src" %}
+#endif
+ins_encode %{
+#ifdef AARCH64
+__ mov($dst$$Register,             ZR, (AsmCondition)($cmp$$cmpcode));
+#else
+__ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
+#endif
+%}
+ins_pipe(ialu_imm);
+%}
+// This instruction also works with CmpN so we don't need cmovPN_reg.
+instruct cmovPI_reg(cmpOp cmp, flagsReg icc, iRegP dst, iRegP src) %{
+match(Set dst (CMoveP (Binary cmp icc) (Binary dst src)));
+ins_cost(150);
+size(4);
+format %{ "MOV$cmp  $dst,$src\t! ptr" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+instruct cmovPI_reg_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegP dst, iRegP src) %{
+match(Set dst (CMoveP (Binary cmp icc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
+_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
+_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
+_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
+ins_cost(150);
+size(4);
+format %{ "MOV$cmp  $dst,$src\t! ptr" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+instruct cmovPIu_reg(cmpOpU cmp, flagsRegU icc, iRegP dst, iRegP src) %{
+match(Set dst (CMoveP (Binary cmp icc) (Binary dst src)));
+ins_cost(150);
+size(4);
+format %{ "MOV$cmp  $dst,$src\t! ptr" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+instruct cmovPI_imm(cmpOp cmp, flagsReg icc, iRegP dst, immP0 src) %{
+match(Set dst (CMoveP (Binary cmp icc) (Binary dst src)));
+ins_cost(140);
+size(4);
+#ifdef AARCH64
+format %{ "MOV$cmp  $dst,ZR\t! ptr" %}
+#else
+format %{ "MOV$cmp  $dst,$src\t! ptr" %}
+#endif
+ins_encode %{
+#ifdef AARCH64
+__ mov($dst$$Register,             ZR, (AsmCondition)($cmp$$cmpcode));
+#else
+__ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
+#endif
+%}
+ins_pipe(ialu_imm);
+%}
+instruct cmovPI_imm_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegP dst, immP0 src) %{
+match(Set dst (CMoveP (Binary cmp icc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
+_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
+_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
+_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
+ins_cost(140);
+size(4);
+#ifdef AARCH64
+format %{ "MOV$cmp  $dst,ZR\t! ptr" %}
+#else
+format %{ "MOV$cmp  $dst,$src\t! ptr" %}
+#endif
+ins_encode %{
+#ifdef AARCH64
+__ mov($dst$$Register,             ZR, (AsmCondition)($cmp$$cmpcode));
+#else
+__ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
+#endif
+%}
+ins_pipe(ialu_imm);
+%}
+instruct cmovPIu_imm(cmpOpU cmp, flagsRegU icc, iRegP dst, immP0 src) %{
+match(Set dst (CMoveP (Binary cmp icc) (Binary dst src)));
+ins_cost(140);
+size(4);
+#ifdef AARCH64
+format %{ "MOV$cmp  $dst,ZR\t! ptr" %}
+#else
+format %{ "MOV$cmp  $dst,$src\t! ptr" %}
+#endif
+ins_encode %{
+#ifdef AARCH64
+__ mov($dst$$Register,             ZR, (AsmCondition)($cmp$$cmpcode));
+#else
+__ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
+#endif
+%}
+ins_pipe(ialu_imm);
+%}
+#ifdef AARCH64
+// Conditional move
+instruct cmovF_reg(cmpOp cmp, flagsReg icc, regF dst, regF src1, regF src2) %{
+match(Set dst (CMoveF (Binary cmp icc) (Binary src2 src1)));
+ins_cost(150);
+size(4);
+format %{ "FCSEL_s $dst,$src1,$src2,$cmp" %}
+ins_encode %{
+__ fcsel_s($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(int_conditional_float_move);
+%}
+instruct cmovD_reg(cmpOp cmp, flagsReg icc, regD dst, regD src1, regD src2) %{
+match(Set dst (CMoveD (Binary cmp icc) (Binary src2 src1)));
+ins_cost(150);
+size(4);
+format %{ "FCSEL_d $dst,$src1,$src2,$cmp" %}
+ins_encode %{
+__ fcsel_d($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(int_conditional_float_move);
+%}
+instruct cmovFP_reg(cmpOpP cmp, flagsRegP icc, regF dst, regF src1, regF src2) %{
+match(Set dst (CMoveF (Binary cmp icc) (Binary src2 src1)));
+ins_cost(150);
+size(4);
+format %{ "FCSEL_s $dst,$src1,$src2,$cmp" %}
+ins_encode %{
+__ fcsel_s($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(int_conditional_float_move);
+%}
+instruct cmovDP_reg(cmpOpP cmp, flagsRegP icc, regD dst, regD src1, regD src2) %{
+match(Set dst (CMoveD (Binary cmp icc) (Binary src2 src1)));
+ins_cost(150);
+size(4);
+format %{ "FCSEL_d $dst,$src1,$src2,$cmp" %}
+ins_encode %{
+__ fcsel_d($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(int_conditional_float_move);
+%}
+instruct cmovFU_reg(cmpOpU cmp, flagsRegU icc, regF dst, regF src1, regF src2) %{
+match(Set dst (CMoveF (Binary cmp icc) (Binary src2 src1)));
+ins_cost(150);
+size(4);
+format %{ "FCSEL_s $dst,$src1,$src2,$cmp" %}
+ins_encode %{
+__ fcsel_s($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(int_conditional_float_move);
+%}
+instruct cmovDU_reg(cmpOpU cmp, flagsRegU icc, regD dst, regD src1, regD src2) %{
+match(Set dst (CMoveD (Binary cmp icc) (Binary src2 src1)));
+ins_cost(150);
+size(4);
+format %{ "FCSEL_d $dst,$src1,$src2,$cmp" %}
+ins_encode %{
+__ fcsel_d($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(int_conditional_float_move);
+%}
+instruct cmovFZ_reg(cmpOp0 cmp, flagsReg_EQNELTGE icc, regF dst, regF src1, regF src2) %{
+match(Set dst (CMoveF (Binary cmp icc) (Binary src2 src1)));
+ins_cost(150);
+size(4);
+format %{ "FCSEL_s $dst,$src1,$src2,$cmp" %}
+ins_encode %{
+__ fcsel_s($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(int_conditional_float_move);
+%}
+instruct cmovDZ_reg(cmpOp0 cmp, flagsReg_EQNELTGE icc, regD dst, regD src1, regD src2) %{
+match(Set dst (CMoveD (Binary cmp icc) (Binary src2 src1)));
+ins_cost(150);
+size(4);
+format %{ "FCSEL_d $dst,$src1,$src2,$cmp" %}
+ins_encode %{
+__ fcsel_d($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(int_conditional_float_move);
+%}
+#else // !AARCH64
+// Conditional move
+instruct cmovFP_reg(cmpOpP cmp, flagsRegP pcc, regF dst, regF src) %{
+match(Set dst (CMoveF (Binary cmp pcc) (Binary dst src)));
+ins_cost(150);
+size(4);
+format %{ "FCPYS$cmp $dst,$src" %}
+ins_encode %{
+__ fcpys($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(int_conditional_float_move);
+%}
+instruct cmovFI_reg(cmpOp cmp, flagsReg icc, regF dst, regF src) %{
+match(Set dst (CMoveF (Binary cmp icc) (Binary dst src)));
+ins_cost(150);
+size(4);
+format %{ "FCPYS$cmp $dst,$src" %}
+ins_encode %{
+__ fcpys($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(int_conditional_float_move);
+%}
+instruct cmovFI_reg_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, regF dst, regF src) %{
+match(Set dst (CMoveF (Binary cmp icc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
+_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
+_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
+_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
+ins_cost(150);
+size(4);
+format %{ "FCPYS$cmp $dst,$src" %}
+ins_encode %{
+__ fcpys($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(int_conditional_float_move);
+%}
+instruct cmovFIu_reg(cmpOpU cmp, flagsRegU icc, regF dst, regF src) %{
+match(Set dst (CMoveF (Binary cmp icc) (Binary dst src)));
+ins_cost(150);
+size(4);
+format %{ "FCPYS$cmp $dst,$src" %}
+ins_encode %{
+__ fcpys($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(int_conditional_float_move);
+%}
+// Conditional move
+instruct cmovDP_reg(cmpOpP cmp, flagsRegP pcc, regD dst, regD src) %{
+match(Set dst (CMoveD (Binary cmp pcc) (Binary dst src)));
+ins_cost(150);
+size(4);
+format %{ "FCPYD$cmp $dst,$src" %}
+ins_encode %{
+__ fcpyd($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(int_conditional_double_move);
+%}
+instruct cmovDI_reg(cmpOp cmp, flagsReg icc, regD dst, regD src) %{
+match(Set dst (CMoveD (Binary cmp icc) (Binary dst src)));
+ins_cost(150);
+size(4);
+format %{ "FCPYD$cmp $dst,$src" %}
+ins_encode %{
+__ fcpyd($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(int_conditional_double_move);
+%}
+instruct cmovDI_reg_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, regD dst, regD src) %{
+match(Set dst (CMoveD (Binary cmp icc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
+ins_cost(150);
+size(4);
+format %{ "FCPYD$cmp $dst,$src" %}
+ins_encode %{
+__ fcpyd($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(int_conditional_double_move);
+%}
+instruct cmovDIu_reg(cmpOpU cmp, flagsRegU icc, regD dst, regD src) %{
+match(Set dst (CMoveD (Binary cmp icc) (Binary dst src)));
+ins_cost(150);
+size(4);
+format %{ "FCPYD$cmp $dst,$src" %}
+ins_encode %{
+__ fcpyd($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(int_conditional_double_move);
+%}
+// Conditional move
+instruct cmovLP_reg(cmpOpP cmp, flagsRegP pcc, iRegL dst, iRegL src) %{
+match(Set dst (CMoveL (Binary cmp pcc) (Binary dst src)));
+ins_cost(150);
+size(8);
+format %{ "MOV$cmp  $dst.lo,$src.lo\t! long\n\t"
+"MOV$cmp  $dst.hi,$src.hi" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
+__ mov($dst$$Register->successor(), $src$$Register->successor(), (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+// TODO: try immLRot2 instead, (0, $con$$constant) becomes
+// (hi($con$$constant), lo($con$$constant)) becomes
+instruct cmovLP_immRot(cmpOpP cmp, flagsRegP pcc, iRegL dst, immLlowRot src) %{
+match(Set dst (CMoveL (Binary cmp pcc) (Binary dst src)));
+ins_cost(140);
+size(8);
+format %{ "MOV$cmp  $dst.lo,$src\t! long\n\t"
+"MOV$cmp  $dst.hi,0" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
+__ mov($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_imm);
+%}
+instruct cmovLP_imm16(cmpOpP cmp, flagsRegP pcc, iRegL dst, immL16 src) %{
+match(Set dst (CMoveL (Binary cmp pcc) (Binary dst src)));
+ins_cost(140);
+size(8);
+format %{ "MOV$cmp  $dst.lo,$src\t! long\n\t"
+"MOV$cmp  $dst.hi,0" %}
+ins_encode %{
+__ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
+__ mov($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_imm);
+%}
+instruct cmovLI_reg(cmpOp cmp, flagsReg icc, iRegL dst, iRegL src) %{
+match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
+ins_cost(150);
+size(8);
+format %{ "MOV$cmp  $dst.lo,$src.lo\t! long\n\t"
+"MOV$cmp  $dst.hi,$src.hi" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
+__ mov($dst$$Register->successor(), $src$$Register->successor(), (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+instruct cmovLI_reg_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegL dst, iRegL src) %{
+match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
+_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
+_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
+_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
+ins_cost(150);
+size(8);
+format %{ "MOV$cmp  $dst.lo,$src.lo\t! long\n\t"
+"MOV$cmp  $dst.hi,$src.hi" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
+__ mov($dst$$Register->successor(), $src$$Register->successor(), (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+// TODO: try immLRot2 instead, (0, $con$$constant) becomes
+// (hi($con$$constant), lo($con$$constant)) becomes
+instruct cmovLI_immRot(cmpOp cmp, flagsReg icc, iRegL dst, immLlowRot src) %{
+match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
+ins_cost(140);
+size(8);
+format %{ "MOV$cmp  $dst.lo,$src\t! long\n\t"
+"MOV$cmp  $dst.hi,0" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
+__ mov($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_imm);
+%}
+// TODO: try immLRot2 instead, (0, $con$$constant) becomes
+// (hi($con$$constant), lo($con$$constant)) becomes
+instruct cmovLI_immRot_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegL dst, immLlowRot src) %{
+match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
+_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
+_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
+_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
+ins_cost(140);
+size(8);
+format %{ "MOV$cmp  $dst.lo,$src\t! long\n\t"
+"MOV$cmp  $dst.hi,0" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
+__ mov($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_imm);
+%}
+instruct cmovLI_imm16(cmpOp cmp, flagsReg icc, iRegL dst, immL16 src) %{
+match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
+ins_cost(140);
+size(8);
+format %{ "MOV$cmp  $dst.lo,$src\t! long\n\t"
+"MOV$cmp  $dst.hi,0" %}
+ins_encode %{
+__ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
+__ movw($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_imm);
+%}
+instruct cmovLI_imm16_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegL dst, immL16 src) %{
+match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
+_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
+_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
+_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
+ins_cost(140);
+size(8);
+format %{ "MOV$cmp  $dst.lo,$src\t! long\n\t"
+"MOV$cmp  $dst.hi,0" %}
+ins_encode %{
+__ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
+__ movw($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_imm);
+%}
+instruct cmovLIu_reg(cmpOpU cmp, flagsRegU icc, iRegL dst, iRegL src) %{
+match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
+ins_cost(150);
+size(8);
+format %{ "MOV$cmp  $dst.lo,$src.lo\t! long\n\t"
+"MOV$cmp  $dst.hi,$src.hi" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
+__ mov($dst$$Register->successor(), $src$$Register->successor(), (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+#endif // !AARCH64
+//----------OS and Locking Instructions----------------------------------------
+// This name is KNOWN by the ADLC and cannot be changed.
+// The ADLC forces a 'TypeRawPtr::BOTTOM' output type
+// for this guy.
+instruct tlsLoadP(RthreadRegP dst) %{
+match(Set dst (ThreadLocal));
+size(0);
+ins_cost(0);
+format %{ "! TLS is in $dst" %}
+ins_encode( /*empty encoding*/ );
+ins_pipe(ialu_none);
+%}
+instruct checkCastPP( iRegP dst ) %{
+match(Set dst (CheckCastPP dst));
+size(0);
+format %{ "! checkcastPP of $dst" %}
+ins_encode( /*empty encoding*/ );
+ins_pipe(empty);
+%}
+instruct castPP( iRegP dst ) %{
+match(Set dst (CastPP dst));
+format %{ "! castPP of $dst" %}
+ins_encode( /*empty encoding*/ );
+ins_pipe(empty);
+%}
+instruct castII( iRegI dst ) %{
+match(Set dst (CastII dst));
+format %{ "! castII of $dst" %}
+ins_encode( /*empty encoding*/ );
+ins_cost(0);
+ins_pipe(empty);
+%}
+//----------Arithmetic Instructions--------------------------------------------
+// Addition Instructions
+// Register Addition
+instruct addI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
+match(Set dst (AddI src1 src2));
+size(4);
+format %{ "add_32 $dst,$src1,$src2\t! int" %}
+ins_encode %{
+__ add_32($dst$$Register, $src1$$Register, $src2$$Register);
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#ifndef AARCH64
+instruct addshlI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
+match(Set dst (AddI (LShiftI src1 src2) src3));
+size(4);
+format %{ "add_32 $dst,$src3,$src1<<$src2\t! int" %}
+ins_encode %{
+__ add_32($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsl, $src2$$Register));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#endif
+#ifdef AARCH64
+#ifdef TODO
+instruct addshlL_reg_imm_reg(iRegL dst, iRegL src1, immU6 src2, iRegL src3) %{
+match(Set dst (AddL (LShiftL src1 src2) src3));
+size(4);
+format %{ "ADD    $dst,$src3,$src1<<$src2\t! long" %}
+ins_encode %{
+__ add($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsl, $src2$$constant));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#endif
+#endif
+instruct addshlI_reg_imm_reg(iRegI dst, iRegI src1, immU5 src2, iRegI src3) %{
+match(Set dst (AddI (LShiftI src1 src2) src3));
+size(4);
+format %{ "add_32 $dst,$src3,$src1<<$src2\t! int" %}
+ins_encode %{
+__ add_32($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsl, $src2$$constant));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#ifndef AARCH64
+instruct addsarI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
+match(Set dst (AddI (RShiftI src1 src2) src3));
+size(4);
+format %{ "add_32 $dst,$src3,$src1>>$src2\t! int" %}
+ins_encode %{
+__ add_32($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, asr, $src2$$Register));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#endif
+instruct addsarI_reg_imm_reg(iRegI dst, iRegI src1, immU5 src2, iRegI src3) %{
+match(Set dst (AddI (RShiftI src1 src2) src3));
+size(4);
+format %{ "add_32 $dst,$src3,$src1>>$src2\t! int" %}
+ins_encode %{
+__ add_32($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, asr, $src2$$constant));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#ifndef AARCH64
+instruct addshrI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
+match(Set dst (AddI (URShiftI src1 src2) src3));
+size(4);
+format %{ "add_32 $dst,$src3,$src1>>>$src2\t! int" %}
+ins_encode %{
+__ add_32($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsr, $src2$$Register));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#endif
+instruct addshrI_reg_imm_reg(iRegI dst, iRegI src1, immU5 src2, iRegI src3) %{
+match(Set dst (AddI (URShiftI src1 src2) src3));
+size(4);
+format %{ "add_32 $dst,$src3,$src1>>>$src2\t! int" %}
+ins_encode %{
+__ add_32($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsr, $src2$$constant));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+// Immediate Addition
+instruct addI_reg_aimmI(iRegI dst, iRegI src1, aimmI src2) %{
+match(Set dst (AddI src1 src2));
+size(4);
+format %{ "add_32 $dst,$src1,$src2\t! int" %}
+ins_encode %{
+__ add_32($dst$$Register, $src1$$Register, $src2$$constant);
+%}
+ins_pipe(ialu_reg_imm);
+%}
+// Pointer Register Addition
+instruct addP_reg_reg(iRegP dst, iRegP src1, iRegX src2) %{
+match(Set dst (AddP src1 src2));
+size(4);
+format %{ "ADD    $dst,$src1,$src2\t! ptr" %}
+ins_encode %{
+__ add($dst$$Register, $src1$$Register, $src2$$Register);
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#ifdef AARCH64
+// unshifted I2L operand
+operand unshiftedI2L(iRegI src2) %{
+//constraint(ALLOC_IN_RC(sp_ptr_reg));
+match(ConvI2L src2);
+op_cost(1);
+format %{ "$src2.w" %}
+interface(MEMORY_INTER) %{
+base($src2);
+index(0xff);
+scale(0x0);
+disp(0x0);
+%}
+%}
+// shifted I2L operand
+operand shiftedI2L(iRegI src2, immI_0_4 src3) %{
+//constraint(ALLOC_IN_RC(sp_ptr_reg));
+match(LShiftX (ConvI2L src2) src3);
+op_cost(1);
+format %{ "$src2.w << $src3" %}
+interface(MEMORY_INTER) %{
+base($src2);
+index(0xff);
+scale($src3);
+disp(0x0);
+%}
+%}
+opclass shiftedRegI(shiftedI2L, unshiftedI2L);
+instruct shlL_reg_regI(iRegL dst, iRegI src1, immU6 src2) %{
+match(Set dst (LShiftL (ConvI2L src1) src2));
+size(4);
+format %{ "LSL    $dst,$src1.w,$src2\t! ptr" %}
+ins_encode %{
+int c = $src2$$constant;
+int r = 64 - c;
+int s = 31;
+if (s >= r) {
+s = r - 1;
+}
+__ sbfm($dst$$Register, $src1$$Register, r, s);
+%}
+ins_pipe(ialu_reg_reg);
+%}
+instruct addP_reg_regI(iRegP dst, iRegP src1, shiftedRegI src2) %{
+match(Set dst (AddP src1 src2));
+ins_cost(DEFAULT_COST * 3/2);
+size(4);
+format %{ "ADD    $dst,$src1,$src2, sxtw\t! ptr" %}
+ins_encode %{
+Register base = reg_to_register_object($src2$$base);
+__ add($dst$$Register, $src1$$Register, base, ex_sxtw, $src2$$scale);
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#endif
+// shifted iRegX operand
+operand shiftedX(iRegX src2, shimmX src3) %{
+//constraint(ALLOC_IN_RC(sp_ptr_reg));
+match(LShiftX src2 src3);
+op_cost(1);
+format %{ "$src2 << $src3" %}
+interface(MEMORY_INTER) %{
+base($src2);
+index(0xff);
+scale($src3);
+disp(0x0);
+%}
+%}
+instruct addshlP_reg_reg_imm(iRegP dst, iRegP src1, shiftedX src2) %{
+match(Set dst (AddP src1 src2));
+ins_cost(DEFAULT_COST * 3/2);
+size(4);
+format %{ "ADD    $dst,$src1,$src2\t! ptr" %}
+ins_encode %{
+Register base = reg_to_register_object($src2$$base);
+__ add($dst$$Register, $src1$$Register, AsmOperand(base, lsl, $src2$$scale));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+// Pointer Immediate Addition
+instruct addP_reg_aimmX(iRegP dst, iRegP src1, aimmX src2) %{
+match(Set dst (AddP src1 src2));
+size(4);
+format %{ "ADD    $dst,$src1,$src2\t! ptr" %}
+ins_encode %{
+__ add($dst$$Register, $src1$$Register, $src2$$constant);
+%}
+ins_pipe(ialu_reg_imm);
+%}
+// Long Addition
+#ifdef AARCH64
+instruct addL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
+match(Set dst (AddL src1 src2));
+size(4);
+format %{ "ADD     $dst,$src1,$src2\t! long" %}
+ins_encode %{
+__ add($dst$$Register, $src1$$Register, $src2$$Register);
+%}
+ins_pipe(ialu_reg_reg);
+%}
+instruct addL_reg_regI(iRegL dst, iRegL src1, shiftedRegI src2) %{
+match(Set dst (AddL src1 src2));
+ins_cost(DEFAULT_COST * 3/2);
+size(4);
+format %{ "ADD    $dst,$src1,$src2, sxtw\t! long" %}
+ins_encode %{
+Register base = reg_to_register_object($src2$$base);
+__ add($dst$$Register, $src1$$Register, base, ex_sxtw, $src2$$scale);
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#else
+instruct addL_reg_reg(iRegL dst, iRegL src1, iRegL src2, flagsReg ccr) %{
+match(Set dst (AddL src1 src2));
+effect(KILL ccr);
+size(8);
+format %{ "ADDS    $dst.lo,$src1.lo,$src2.lo\t! long\n\t"
+"ADC     $dst.hi,$src1.hi,$src2.hi" %}
+ins_encode %{
+__ adds($dst$$Register, $src1$$Register, $src2$$Register);
+__ adc($dst$$Register->successor(), $src1$$Register->successor(), $src2$$Register->successor());
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#endif
+#ifdef AARCH64
+// Immediate Addition
+instruct addL_reg_aimm(iRegL dst, iRegL src1, aimmL src2) %{
+match(Set dst (AddL src1 src2));
+size(4);
+format %{ "ADD    $dst,$src1,$src2\t! long" %}
+ins_encode %{
+__ add($dst$$Register, $src1$$Register, $src2$$constant);
+%}
+ins_pipe(ialu_reg_imm);
+%}
+instruct addL_reg_immLneg(iRegL dst, iRegL src1, aimmLneg src2) %{
+match(Set dst (SubL src1 src2));
+size(4);
+format %{ "ADD    $dst,$src1,-($src2)\t! long" %}
+ins_encode %{
+__ add($dst$$Register, $src1$$Register, -$src2$$constant);
+%}
+ins_pipe(ialu_reg_imm);
+%}
+#else
+// TODO
+#endif
+#ifndef AARCH64
+// TODO: try immLRot2 instead, (0, $con$$constant) becomes
+// (hi($con$$constant), lo($con$$constant)) becomes
+instruct addL_reg_immRot(iRegL dst, iRegL src1, immLlowRot con, flagsReg ccr) %{
+match(Set dst (AddL src1 con));
+effect(KILL ccr);
+size(8);
+format %{ "ADDS    $dst.lo,$src1.lo,$con\t! long\n\t"
+"ADC     $dst.hi,$src1.hi,0" %}
+ins_encode %{
+__ adds($dst$$Register, $src1$$Register, $con$$constant);
+__ adc($dst$$Register->successor(), $src1$$Register->successor(), 0);
+%}
+ins_pipe(ialu_reg_imm);
+%}
+#endif
+//----------Conditional_store--------------------------------------------------
+// Conditional-store of the updated heap-top.
+// Used during allocation of the shared heap.
+// Sets flags (EQ) on success.
+// TODO: optimize out barriers with AArch64 load-acquire/store-release
+// LoadP-locked.
+instruct loadPLocked(iRegP dst, memoryex mem) %{
+match(Set dst (LoadPLocked mem));
+size(4);
+format %{ "LDREX  $dst,$mem" %}
+ins_encode %{
+#ifdef AARCH64
+Register base = reg_to_register_object($mem$$base);
+__ ldxr($dst$$Register, base);
+#else
+__ ldrex($dst$$Register,$mem$$Address);
+#endif
+%}
+ins_pipe(iload_mem);
+%}
+instruct storePConditional( memoryex heap_top_ptr, iRegP oldval, iRegP newval, iRegI tmp, flagsRegP pcc ) %{
+predicate(_kids[1]->_kids[0]->_leaf->Opcode() == Op_LoadPLocked); // only works in conjunction with a LoadPLocked node
+match(Set pcc (StorePConditional heap_top_ptr (Binary oldval newval)));
+effect( TEMP tmp );
+size(8);
+format %{ "STREX  $tmp,$newval,$heap_top_ptr\n\t"
+"CMP    $tmp, 0" %}
+ins_encode %{
+#ifdef AARCH64
+Register base = reg_to_register_object($heap_top_ptr$$base);
+__ stxr($tmp$$Register, $newval$$Register, base);
+#else
+__ strex($tmp$$Register, $newval$$Register, $heap_top_ptr$$Address);
+#endif
+__ cmp($tmp$$Register, 0);
+%}
+ins_pipe( long_memory_op );
+%}
+// Conditional-store of an intx value.
+instruct storeXConditional( memoryex mem, iRegX oldval, iRegX newval, iRegX tmp, flagsReg icc ) %{
+#ifdef AARCH64
+match(Set icc (StoreLConditional mem (Binary oldval newval)));
+effect( TEMP tmp );
+size(28);
+format %{ "loop:\n\t"
+"LDXR     $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem], DOESN'T set $newval=[$mem] in any case\n\t"
+"SUBS     $tmp, $tmp, $oldval\n\t"
+"B.ne     done\n\t"
+"STXR     $tmp, $newval, $mem\n\t"
+"CBNZ_w   $tmp, loop\n\t"
+"CMP      $tmp, 0\n\t"
+"done:\n\t"
+"membar   LoadStore|LoadLoad" %}
+#else
+match(Set icc (StoreIConditional mem (Binary oldval newval)));
+effect( TEMP tmp );
+size(28);
+format %{ "loop: \n\t"
+"LDREX    $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem], DOESN'T set $newval=[$mem] in any case\n\t"
+"XORS     $tmp,$tmp, $oldval\n\t"
+"STREX.eq $tmp, $newval, $mem\n\t"
+"CMP.eq   $tmp, 1 \n\t"
+"B.eq     loop \n\t"
+"TEQ      $tmp, 0\n\t"
+"membar   LoadStore|LoadLoad" %}
+#endif
+ins_encode %{
+Label loop;
+__ bind(loop);
+#ifdef AARCH64
+// FIXME: use load-acquire/store-release, remove membar?
+Label done;
+Register base = reg_to_register_object($mem$$base);
+__ ldxr($tmp$$Register, base);
+__ subs($tmp$$Register, $tmp$$Register, $oldval$$Register);
+__ b(done, ne);
+__ stxr($tmp$$Register, $newval$$Register, base);
+__ cbnz_w($tmp$$Register, loop);
+__ cmp($tmp$$Register, 0);
+__ bind(done);
+#else
+__ ldrex($tmp$$Register, $mem$$Address);
+__ eors($tmp$$Register, $tmp$$Register, $oldval$$Register);
+__ strex($tmp$$Register, $newval$$Register, $mem$$Address, eq);
+__ cmp($tmp$$Register, 1, eq);
+__ b(loop, eq);
+__ teq($tmp$$Register, 0);
+#endif
+// used by biased locking only. Requires a membar.
+__ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::LoadStore | MacroAssembler::LoadLoad), noreg);
+%}
+ins_pipe( long_memory_op );
+%}
+// No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
+#ifdef AARCH64
+// TODO: if combined with membar, elide membar and use
+// load-acquire/store-release if appropriate
+instruct compareAndSwapL_bool(memoryex mem, iRegL oldval, iRegL newval, iRegI res, iRegI tmp, flagsReg ccr) %{
+match(Set res (CompareAndSwapL mem (Binary oldval newval)));
+effect( KILL ccr, TEMP tmp);
+size(24);
+format %{ "loop:\n\t"
+"LDXR     $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
+"CMP      $tmp, $oldval\n\t"
+"B.ne     done\n\t"
+"STXR     $tmp, $newval, $mem\n\t"
+"CBNZ_w   $tmp, loop\n\t"
+"done:\n\t"
+"CSET_w   $res, eq" %}
+ins_encode %{
+Register base = reg_to_register_object($mem$$base);
+Label loop, done;
+__ bind(loop);
+__ ldxr($tmp$$Register, base);
+__ cmp($tmp$$Register, $oldval$$Register);
+__ b(done, ne);
+__ stxr($tmp$$Register, $newval$$Register, base);
+__ cbnz_w($tmp$$Register, loop);
+__ bind(done);
+__ cset_w($res$$Register, eq);
+%}
+ins_pipe( long_memory_op );
+%}
+instruct compareAndSwapI_bool(memoryex mem, iRegI oldval, iRegI newval, iRegI res, iRegI tmp, flagsReg ccr) %{
+match(Set res (CompareAndSwapI mem (Binary oldval newval)));
+effect( KILL ccr, TEMP tmp);
+size(24);
+format %{ "loop:\n\t"
+"LDXR_w   $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
+"CMP_w    $tmp, $oldval\n\t"
+"B.ne     done\n\t"
+"STXR_w   $tmp, $newval, $mem\n\t"
+"CBNZ_w   $tmp, loop\n\t"
+"done:\n\t"
+"CSET_w   $res, eq" %}
+ins_encode %{
+Register base = reg_to_register_object($mem$$base);
+Label loop, done;
+__ bind(loop);
+__ ldxr_w($tmp$$Register, base);
+__ cmp_w($tmp$$Register, $oldval$$Register);
+__ b(done, ne);
+__ stxr_w($tmp$$Register, $newval$$Register,  base);
+__ cbnz_w($tmp$$Register, loop);
+__ bind(done);
+__ cset_w($res$$Register, eq);
+%}
+ins_pipe( long_memory_op );
+%}
+// tmp must use iRegI instead of iRegN until 8051805 is fixed.
+instruct compareAndSwapN_bool(memoryex mem, iRegN oldval, iRegN newval, iRegI res, iRegI tmp, flagsReg ccr) %{
+match(Set res (CompareAndSwapN mem (Binary oldval newval)));
+effect( KILL ccr, TEMP tmp);
+size(24);
+format %{ "loop:\n\t"
+"LDXR_w   $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
+"CMP_w    $tmp, $oldval\n\t"
+"B.ne     done\n\t"
+"STXR_w   $tmp, $newval, $mem\n\t"
+"CBNZ_w   $tmp, loop\n\t"
+"done:\n\t"
+"CSET_w   $res, eq" %}
+ins_encode %{
+Register base = reg_to_register_object($mem$$base);
+Label loop, done;
+__ bind(loop);
+__ ldxr_w($tmp$$Register, base);
+__ cmp_w($tmp$$Register, $oldval$$Register);
+__ b(done, ne);
+__ stxr_w($tmp$$Register, $newval$$Register,  base);
+__ cbnz_w($tmp$$Register, loop);
+__ bind(done);
+__ cset_w($res$$Register, eq);
+%}
+ins_pipe( long_memory_op );
+%}
+instruct compareAndSwapP_bool(memoryex mem, iRegP oldval, iRegP newval, iRegI res, iRegI tmp, flagsReg ccr) %{
+match(Set res (CompareAndSwapP mem (Binary oldval newval)));
+effect( KILL ccr, TEMP tmp);
+size(24);
+format %{ "loop:\n\t"
+"LDXR     $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
+"CMP      $tmp, $oldval\n\t"
+"B.ne     done\n\t"
+"STXR     $tmp, $newval, $mem\n\t"
+"CBNZ_w   $tmp, loop\n\t"
+"done:\n\t"
+"CSET_w   $res, eq" %}
+ins_encode %{
+Register base = reg_to_register_object($mem$$base);
+Label loop, done;
+__ bind(loop);
+__ ldxr($tmp$$Register, base);
+__ cmp($tmp$$Register, $oldval$$Register);
+__ b(done, ne);
+__ stxr($tmp$$Register, $newval$$Register,  base);
+__ cbnz_w($tmp$$Register, loop);
+__ bind(done);
+__ cset_w($res$$Register, eq);
+%}
+ins_pipe( long_memory_op );
+%}
+#else // !AARCH64
+instruct compareAndSwapL_bool(memoryex mem, iRegL oldval, iRegLd newval, iRegI res, iRegLd tmp, flagsReg ccr ) %{
+match(Set res (CompareAndSwapL mem (Binary oldval newval)));
+effect( KILL ccr, TEMP tmp);
+size(32);
+format %{ "loop: \n\t"
+"LDREXD   $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
+"CMP      $tmp.lo, $oldval.lo\n\t"
+"CMP.eq   $tmp.hi, $oldval.hi\n\t"
+"STREXD.eq $tmp, $newval, $mem\n\t"
+"MOV.ne   $tmp, 0 \n\t"
+"XORS.eq  $tmp,$tmp, 1 \n\t"
+"B.eq     loop \n\t"
+"MOV      $res, $tmp" %}
+ins_encode %{
+Label loop;
+__ bind(loop);
+__ ldrexd($tmp$$Register, $mem$$Address);
+__ cmp($tmp$$Register, $oldval$$Register);
+__ cmp($tmp$$Register->successor(), $oldval$$Register->successor(), eq);
+__ strexd($tmp$$Register, $newval$$Register, $mem$$Address, eq);
+__ mov($tmp$$Register, 0, ne);
+__ eors($tmp$$Register, $tmp$$Register, 1, eq);
+__ b(loop, eq);
+__ mov($res$$Register, $tmp$$Register);
+%}
+ins_pipe( long_memory_op );
+%}
+instruct compareAndSwapI_bool(memoryex mem, iRegI oldval, iRegI newval, iRegI res, iRegI tmp, flagsReg ccr ) %{
+match(Set res (CompareAndSwapI mem (Binary oldval newval)));
+effect( KILL ccr, TEMP tmp);
+size(28);
+format %{ "loop: \n\t"
+"LDREX    $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
+"CMP      $tmp, $oldval\n\t"
+"STREX.eq $tmp, $newval, $mem\n\t"
+"MOV.ne   $tmp, 0 \n\t"
+"XORS.eq  $tmp,$tmp, 1 \n\t"
+"B.eq     loop \n\t"
+"MOV      $res, $tmp" %}
+ins_encode %{
+Label loop;
+__ bind(loop);
+__ ldrex($tmp$$Register,$mem$$Address);
+__ cmp($tmp$$Register, $oldval$$Register);
+__ strex($tmp$$Register, $newval$$Register, $mem$$Address, eq);
+__ mov($tmp$$Register, 0, ne);
+__ eors($tmp$$Register, $tmp$$Register, 1, eq);
+__ b(loop, eq);
+__ mov($res$$Register, $tmp$$Register);
+%}
+ins_pipe( long_memory_op );
+%}
+instruct compareAndSwapP_bool(memoryex mem, iRegP oldval, iRegP newval, iRegI res, iRegI tmp, flagsReg ccr ) %{
+match(Set res (CompareAndSwapP mem (Binary oldval newval)));
+effect( KILL ccr, TEMP tmp);
+size(28);
+format %{ "loop: \n\t"
+"LDREX    $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
+"CMP      $tmp, $oldval\n\t"
+"STREX.eq $tmp, $newval, $mem\n\t"
+"MOV.ne   $tmp, 0 \n\t"
+"EORS.eq  $tmp,$tmp, 1 \n\t"
+"B.eq     loop \n\t"
+"MOV      $res, $tmp" %}
+ins_encode %{
+Label loop;
+__ bind(loop);
+__ ldrex($tmp$$Register,$mem$$Address);
+__ cmp($tmp$$Register, $oldval$$Register);
+__ strex($tmp$$Register, $newval$$Register, $mem$$Address, eq);
+__ mov($tmp$$Register, 0, ne);
+__ eors($tmp$$Register, $tmp$$Register, 1, eq);
+__ b(loop, eq);
+__ mov($res$$Register, $tmp$$Register);
+%}
+ins_pipe( long_memory_op );
+%}
+#endif // !AARCH64
+#ifdef AARCH64
+instruct xaddI_aimmI_no_res(memoryex mem, aimmI add, Universe dummy, iRegI tmp1, iRegI tmp2) %{
+predicate(n->as_LoadStore()->result_not_used());
+match(Set dummy (GetAndAddI mem add));
+effect(TEMP tmp1, TEMP tmp2);
+size(16);
+format %{ "loop:\n\t"
+"LDXR_w   $tmp1, $mem\n\t"
+"ADD_w    $tmp1, $tmp1, $add\n\t"
+"STXR_w   $tmp2, $tmp1, $mem\n\t"
+"CBNZ_w   $tmp2, loop" %}
+ins_encode %{
+Label loop;
+Register base = reg_to_register_object($mem$$base);
+__ bind(loop);
+__ ldxr_w($tmp1$$Register, base);
+__ add_w($tmp1$$Register, $tmp1$$Register, $add$$constant);
+__ stxr_w($tmp2$$Register, $tmp1$$Register, base);
+__ cbnz_w($tmp2$$Register, loop);
+%}
+ins_pipe( long_memory_op );
+%}
+#else
+instruct xaddI_aimmI_no_res(memoryex mem, aimmI add, Universe dummy, iRegI tmp1, iRegI tmp2, flagsReg ccr) %{
+predicate(n->as_LoadStore()->result_not_used());
+match(Set dummy (GetAndAddI mem add));
+effect(KILL ccr, TEMP tmp1, TEMP tmp2);
+size(20);
+format %{ "loop: \n\t"
+"LDREX    $tmp1, $mem\n\t"
+"ADD      $tmp1, $tmp1, $add\n\t"
+"STREX    $tmp2, $tmp1, $mem\n\t"
+"CMP      $tmp2, 0 \n\t"
+"B.ne     loop \n\t" %}
+ins_encode %{
+Label loop;
+__ bind(loop);
+__ ldrex($tmp1$$Register,$mem$$Address);
+__ add($tmp1$$Register, $tmp1$$Register, $add$$constant);
+__ strex($tmp2$$Register, $tmp1$$Register, $mem$$Address);
+__ cmp($tmp2$$Register, 0);
+__ b(loop, ne);
+%}
+ins_pipe( long_memory_op );
+%}
+#endif
+#ifdef AARCH64
+instruct xaddI_reg_no_res(memoryex mem, iRegI add, Universe dummy, iRegI tmp1, iRegI tmp2) %{
+predicate(n->as_LoadStore()->result_not_used());
+match(Set dummy (GetAndAddI mem add));
+effect(TEMP tmp1, TEMP tmp2);
+size(16);
+format %{ "loop:\n\t"
+"LDXR_w   $tmp1, $mem\n\t"
+"ADD_w    $tmp1, $tmp1, $add\n\t"
+"STXR_w   $tmp2, $tmp1, $mem\n\t"
+"CBNZ_w   $tmp2, loop" %}
+ins_encode %{
+Label loop;
+Register base = reg_to_register_object($mem$$base);
+__ bind(loop);
+__ ldxr_w($tmp1$$Register, base);
+__ add_w($tmp1$$Register, $tmp1$$Register, $add$$Register);
+__ stxr_w($tmp2$$Register, $tmp1$$Register, base);
+__ cbnz_w($tmp2$$Register, loop);
+%}
+ins_pipe( long_memory_op );
+%}
+#else
+instruct xaddI_reg_no_res(memoryex mem, iRegI add, Universe dummy, iRegI tmp1, iRegI tmp2, flagsReg ccr) %{
+predicate(n->as_LoadStore()->result_not_used());
+match(Set dummy (GetAndAddI mem add));
+effect(KILL ccr, TEMP tmp1, TEMP tmp2);
+size(20);
+format %{ "loop: \n\t"
+"LDREX    $tmp1, $mem\n\t"
+"ADD      $tmp1, $tmp1, $add\n\t"
+"STREX    $tmp2, $tmp1, $mem\n\t"
+"CMP      $tmp2, 0 \n\t"
+"B.ne     loop \n\t" %}
+ins_encode %{
+Label loop;
+__ bind(loop);
+__ ldrex($tmp1$$Register,$mem$$Address);
+__ add($tmp1$$Register, $tmp1$$Register, $add$$Register);
+__ strex($tmp2$$Register, $tmp1$$Register, $mem$$Address);
+__ cmp($tmp2$$Register, 0);
+__ b(loop, ne);
+%}
+ins_pipe( long_memory_op );
+%}
+#endif
+#ifdef AARCH64
+instruct xaddI_aimmI(memoryex mem, aimmI add, iRegI res, iRegI tmp1, iRegI tmp2) %{
+match(Set res (GetAndAddI mem add));
+effect(TEMP tmp1, TEMP tmp2, TEMP res);
+size(16);
+format %{ "loop:\n\t"
+"LDXR_w   $res, $mem\n\t"
+"ADD_w    $tmp1, $res, $add\n\t"
+"STXR_w   $tmp2, $tmp1, $mem\n\t"
+"CBNZ_w   $tmp2, loop" %}
+ins_encode %{
+Label loop;
+Register base = reg_to_register_object($mem$$base);
+__ bind(loop);
+__ ldxr_w($res$$Register, base);
+__ add_w($tmp1$$Register, $res$$Register, $add$$constant);
+__ stxr_w($tmp2$$Register, $tmp1$$Register, base);
+__ cbnz_w($tmp2$$Register, loop);
+%}
+ins_pipe( long_memory_op );
+%}
+#else
+instruct xaddI_aimmI(memoryex mem, aimmI add, iRegI res, iRegI tmp1, iRegI tmp2, flagsReg ccr) %{
+match(Set res (GetAndAddI mem add));
+effect(KILL ccr, TEMP tmp1, TEMP tmp2, TEMP res);
+size(20);
+format %{ "loop: \n\t"
+"LDREX    $res, $mem\n\t"
+"ADD      $tmp1, $res, $add\n\t"
+"STREX    $tmp2, $tmp1, $mem\n\t"
+"CMP      $tmp2, 0 \n\t"
+"B.ne     loop \n\t" %}
+ins_encode %{
+Label loop;
+__ bind(loop);
+__ ldrex($res$$Register,$mem$$Address);
+__ add($tmp1$$Register, $res$$Register, $add$$constant);
+__ strex($tmp2$$Register, $tmp1$$Register, $mem$$Address);
+__ cmp($tmp2$$Register, 0);
+__ b(loop, ne);
+%}
+ins_pipe( long_memory_op );
+%}
+#endif
+#ifdef AARCH64
+instruct xaddI_reg(memoryex mem, iRegI add, iRegI res, iRegI tmp1, iRegI tmp2) %{
+match(Set res (GetAndAddI mem add));
+effect(TEMP tmp1, TEMP tmp2, TEMP res);
+size(16);
+format %{ "loop:\n\t"
+"LDXR_w   $res, $mem\n\t"
+"ADD_w    $tmp1, $res, $add\n\t"
+"STXR_w   $tmp2, $tmp1, $mem\n\t"
+"CBNZ_w   $tmp2, loop" %}
+ins_encode %{
+Label loop;
+Register base = reg_to_register_object($mem$$base);
+__ bind(loop);
+__ ldxr_w($res$$Register, base);
+__ add_w($tmp1$$Register, $res$$Register, $add$$Register);
+__ stxr_w($tmp2$$Register, $tmp1$$Register, base);
+__ cbnz_w($tmp2$$Register, loop);
+%}
+ins_pipe( long_memory_op );
+%}
+#else
+instruct xaddI_reg(memoryex mem, iRegI add, iRegI res, iRegI tmp1, iRegI tmp2, flagsReg ccr) %{
+match(Set res (GetAndAddI mem add));
+effect(KILL ccr, TEMP tmp1, TEMP tmp2, TEMP res);
+size(20);
+format %{ "loop: \n\t"
+"LDREX    $res, $mem\n\t"
+"ADD      $tmp1, $res, $add\n\t"
+"STREX    $tmp2, $tmp1, $mem\n\t"
+"CMP      $tmp2, 0 \n\t"
+"B.ne     loop \n\t" %}
+ins_encode %{
+Label loop;
+__ bind(loop);
+__ ldrex($res$$Register,$mem$$Address);
+__ add($tmp1$$Register, $res$$Register, $add$$Register);
+__ strex($tmp2$$Register, $tmp1$$Register, $mem$$Address);
+__ cmp($tmp2$$Register, 0);
+__ b(loop, ne);
+%}
+ins_pipe( long_memory_op );
+%}
+#endif
+#ifdef AARCH64
+instruct xaddL_reg_no_res(memoryex mem, iRegL add, Universe dummy, iRegL tmp1, iRegI tmp2) %{
+predicate(n->as_LoadStore()->result_not_used());
+match(Set dummy (GetAndAddL mem add));
+effect(TEMP tmp1, TEMP tmp2);
+size(16);
+format %{ "loop:\n\t"
+"LDXR     $tmp1, $mem\n\t"
+"ADD      $tmp1, $tmp1, $add\n\t"
+"STXR     $tmp2, $tmp1, $mem\n\t"
+"CBNZ_w   $tmp2, loop" %}
+ins_encode %{
+Label loop;
+Register base = reg_to_register_object($mem$$base);
+__ bind(loop);
+__ ldxr($tmp1$$Register, base);
+__ add($tmp1$$Register, $tmp1$$Register, $add$$Register);
+__ stxr($tmp2$$Register, $tmp1$$Register, base);
+__ cbnz_w($tmp2$$Register, loop);
+%}
+ins_pipe( long_memory_op );
+%}
+#else
+instruct xaddL_reg_no_res(memoryex mem, iRegL add, Universe dummy, iRegLd tmp1, iRegI tmp2, flagsReg ccr) %{
+predicate(n->as_LoadStore()->result_not_used());
+match(Set dummy (GetAndAddL mem add));
+effect( KILL ccr, TEMP tmp1, TEMP tmp2);
+size(24);
+format %{ "loop: \n\t"
+"LDREXD   $tmp1, $mem\n\t"
+"ADDS     $tmp1.lo, $tmp1.lo, $add.lo\n\t"
+"ADC      $tmp1.hi, $tmp1.hi, $add.hi\n\t"
+"STREXD   $tmp2, $tmp1, $mem\n\t"
+"CMP      $tmp2, 0 \n\t"
+"B.ne     loop \n\t" %}
+ins_encode %{
+Label loop;
+__ bind(loop);
+__ ldrexd($tmp1$$Register, $mem$$Address);
+__ adds($tmp1$$Register, $tmp1$$Register, $add$$Register);
+__ adc($tmp1$$Register->successor(), $tmp1$$Register->successor(), $add$$Register->successor());
+__ strexd($tmp2$$Register, $tmp1$$Register, $mem$$Address);
+__ cmp($tmp2$$Register, 0);
+__ b(loop, ne);
+%}
+ins_pipe( long_memory_op );
+%}
+#endif
+#ifdef AARCH64
+instruct xaddL_imm_no_res(memoryex mem, aimmL add, Universe dummy, iRegL tmp1, iRegI tmp2) %{
+predicate(n->as_LoadStore()->result_not_used());
+match(Set dummy (GetAndAddL mem add));
+effect(TEMP tmp1, TEMP tmp2);
+size(16);
+format %{ "loop:\n\t"
+"LDXR     $tmp1, $mem\n\t"
+"ADD      $tmp1, $tmp1, $add\n\t"
+"STXR     $tmp2, $tmp1, $mem\n\t"
+"CBNZ_w   $tmp2, loop" %}
+ins_encode %{
+Label loop;
+Register base = reg_to_register_object($mem$$base);
+__ bind(loop);
+__ ldxr($tmp1$$Register, base);
+__ add($tmp1$$Register, $tmp1$$Register, $add$$constant);
+__ stxr($tmp2$$Register, $tmp1$$Register, base);
+__ cbnz_w($tmp2$$Register, loop);
+%}
+ins_pipe( long_memory_op );
+%}
+#else
+// TODO: try immLRot2 instead, (0, $con$$constant) becomes
+// (hi($con$$constant), lo($con$$constant)) becomes
+instruct xaddL_immRot_no_res(memoryex mem, immLlowRot add, Universe dummy, iRegLd tmp1, iRegI tmp2, flagsReg ccr) %{
+predicate(n->as_LoadStore()->result_not_used());
+match(Set dummy (GetAndAddL mem add));
+effect( KILL ccr, TEMP tmp1, TEMP tmp2);
+size(24);
+format %{ "loop: \n\t"
+"LDREXD   $tmp1, $mem\n\t"
+"ADDS     $tmp1.lo, $tmp1.lo, $add\n\t"
+"ADC      $tmp1.hi, $tmp1.hi, 0\n\t"
+"STREXD   $tmp2, $tmp1, $mem\n\t"
+"CMP      $tmp2, 0 \n\t"
+"B.ne     loop \n\t" %}
+ins_encode %{
+Label loop;
+__ bind(loop);
+__ ldrexd($tmp1$$Register, $mem$$Address);
+__ adds($tmp1$$Register, $tmp1$$Register, $add$$constant);
+__ adc($tmp1$$Register->successor(), $tmp1$$Register->successor(), 0);
+__ strexd($tmp2$$Register, $tmp1$$Register, $mem$$Address);
+__ cmp($tmp2$$Register, 0);
+__ b(loop, ne);
+%}
+ins_pipe( long_memory_op );
+%}
+#endif
+#ifdef AARCH64
+instruct xaddL_reg(memoryex mem, iRegL add, iRegL res, iRegL tmp1, iRegI tmp2) %{
+match(Set res (GetAndAddL mem add));
+effect(TEMP tmp1, TEMP tmp2, TEMP res);
+size(16);
+format %{ "loop:\n\t"
+"LDXR     $res, $mem\n\t"
+"ADD      $tmp1, $res, $add\n\t"
+"STXR     $tmp2, $tmp1, $mem\n\t"
+"CBNZ_w   $tmp2, loop" %}
+ins_encode %{
+Label loop;
+Register base = reg_to_register_object($mem$$base);
+__ bind(loop);
+__ ldxr($res$$Register, base);
+__ add($tmp1$$Register, $res$$Register, $add$$Register);
+__ stxr($tmp2$$Register, $tmp1$$Register, base);
+__ cbnz_w($tmp2$$Register, loop);
+%}
+ins_pipe( long_memory_op );
+%}
+#else
+instruct xaddL_reg(memoryex mem, iRegL add, iRegLd res, iRegLd tmp1, iRegI tmp2, flagsReg ccr) %{
+match(Set res (GetAndAddL mem add));
+effect( KILL ccr, TEMP tmp1, TEMP tmp2, TEMP res);
+size(24);
+format %{ "loop: \n\t"
+"LDREXD   $res, $mem\n\t"
+"ADDS     $tmp1.lo, $res.lo, $add.lo\n\t"
+"ADC      $tmp1.hi, $res.hi, $add.hi\n\t"
+"STREXD   $tmp2, $tmp1, $mem\n\t"
+"CMP      $tmp2, 0 \n\t"
+"B.ne     loop \n\t" %}
+ins_encode %{
+Label loop;
+__ bind(loop);
+__ ldrexd($res$$Register, $mem$$Address);
+__ adds($tmp1$$Register, $res$$Register, $add$$Register);
+__ adc($tmp1$$Register->successor(), $res$$Register->successor(), $add$$Register->successor());
+__ strexd($tmp2$$Register, $tmp1$$Register, $mem$$Address);
+__ cmp($tmp2$$Register, 0);
+__ b(loop, ne);
+%}
+ins_pipe( long_memory_op );
+%}
+#endif
+#ifdef AARCH64
+instruct xaddL_imm(memoryex mem, aimmL add, iRegL res, iRegL tmp1, iRegI tmp2) %{
+match(Set res (GetAndAddL mem add));
+effect(TEMP tmp1, TEMP tmp2, TEMP res);
+size(16);
+format %{ "loop:\n\t"
+"LDXR     $res, $mem\n\t"
+"ADD      $tmp1, $res, $add\n\t"
+"STXR     $tmp2, $tmp1, $mem\n\t"
+"CBNZ_w   $tmp2, loop" %}
+ins_encode %{
+Label loop;
+Register base = reg_to_register_object($mem$$base);
+__ bind(loop);
+__ ldxr($res$$Register, base);
+__ add($tmp1$$Register, $res$$Register, $add$$constant);
+__ stxr($tmp2$$Register, $tmp1$$Register, base);
+__ cbnz_w($tmp2$$Register, loop);
+%}
+ins_pipe( long_memory_op );
+%}
+#else
+// TODO: try immLRot2 instead, (0, $con$$constant) becomes
+// (hi($con$$constant), lo($con$$constant)) becomes
+instruct xaddL_immRot(memoryex mem, immLlowRot add, iRegLd res, iRegLd tmp1, iRegI tmp2, flagsReg ccr) %{
+match(Set res (GetAndAddL mem add));
+effect( KILL ccr, TEMP tmp1, TEMP tmp2, TEMP res);
+size(24);
+format %{ "loop: \n\t"
+"LDREXD   $res, $mem\n\t"
+"ADDS     $tmp1.lo, $res.lo, $add\n\t"
+"ADC      $tmp1.hi, $res.hi, 0\n\t"
+"STREXD   $tmp2, $tmp1, $mem\n\t"
+"CMP      $tmp2, 0 \n\t"
+"B.ne     loop \n\t" %}
+ins_encode %{
+Label loop;
+__ bind(loop);
+__ ldrexd($res$$Register, $mem$$Address);
+__ adds($tmp1$$Register, $res$$Register, $add$$constant);
+__ adc($tmp1$$Register->successor(), $res$$Register->successor(), 0);
+__ strexd($tmp2$$Register, $tmp1$$Register, $mem$$Address);
+__ cmp($tmp2$$Register, 0);
+__ b(loop, ne);
+%}
+ins_pipe( long_memory_op );
+%}
+#endif
+#ifdef AARCH64
+instruct xchgI(memoryex mem, iRegI newval, iRegI res, iRegI tmp) %{
+match(Set res (GetAndSetI mem newval));
+effect(TEMP tmp, TEMP res);
+size(12);
+format %{ "loop:\n\t"
+"LDXR_w   $res, $mem\n\t"
+"STXR_w   $tmp, $newval, $mem\n\t"
+"CBNZ_w   $tmp, loop" %}
+ins_encode %{
+Label loop;
+Register base = reg_to_register_object($mem$$base);
+__ bind(loop);
+__ ldxr_w($res$$Register, base);
+__ stxr_w($tmp$$Register, $newval$$Register, base);
+__ cbnz_w($tmp$$Register, loop);
+%}
+ins_pipe( long_memory_op );
+%}
+#ifdef XXX
+// Disabled until 8051805 is fixed.
+instruct xchgN(memoryex mem, iRegN newval, iRegN res, iRegN tmp) %{
+match(Set res (GetAndSetN mem newval));
+effect(TEMP tmp, TEMP res);
+size(12);
+format %{ "loop:\n\t"
+"LDXR_w   $res, $mem\n\t"
+"STXR_w   $tmp, $newval, $mem\n\t"
+"CBNZ_w   $tmp, loop" %}
+ins_encode %{
+Label loop;
+Register base = reg_to_register_object($mem$$base);
+__ bind(loop);
+__ ldxr_w($res$$Register, base);
+__ stxr_w($tmp$$Register, $newval$$Register, base);
+__ cbnz_w($tmp$$Register, loop);
+%}
+ins_pipe( long_memory_op );
+%}
+#endif
+#else
+instruct xchgI(memoryex mem, iRegI newval, iRegI res, iRegI tmp, flagsReg ccr) %{
+match(Set res (GetAndSetI mem newval));
+effect(KILL ccr, TEMP tmp, TEMP res);
+size(16);
+format %{ "loop: \n\t"
+"LDREX    $res, $mem\n\t"
+"STREX    $tmp, $newval, $mem\n\t"
+"CMP      $tmp, 0 \n\t"
+"B.ne     loop \n\t" %}
+ins_encode %{
+Label loop;
+__ bind(loop);
+__ ldrex($res$$Register,$mem$$Address);
+__ strex($tmp$$Register, $newval$$Register, $mem$$Address);
+__ cmp($tmp$$Register, 0);
+__ b(loop, ne);
+%}
+ins_pipe( long_memory_op );
+%}
+#endif
+#ifdef AARCH64
+instruct xchgL(memoryex mem, iRegL newval, iRegL res, iRegI tmp) %{
+match(Set res (GetAndSetL mem newval));
+effect(TEMP tmp, TEMP res);
+size(12);
+format %{ "loop:\n\t"
+"LDXR     $res, $mem\n\t"
+"STXR     $tmp, $newval, $mem\n\t"
+"CBNZ_w   $tmp, loop" %}
+ins_encode %{
+Label loop;
+Register base = reg_to_register_object($mem$$base);
+__ bind(loop);
+__ ldxr($res$$Register, base);
+__ stxr($tmp$$Register, $newval$$Register, base);
+__ cbnz_w($tmp$$Register, loop);
+%}
+ins_pipe( long_memory_op );
+%}
+#else
+instruct xchgL(memoryex mem, iRegLd newval, iRegLd res, iRegI tmp, flagsReg ccr) %{
+match(Set res (GetAndSetL mem newval));
+effect( KILL ccr, TEMP tmp, TEMP res);
+size(16);
+format %{ "loop: \n\t"
+"LDREXD   $res, $mem\n\t"
+"STREXD   $tmp, $newval, $mem\n\t"
+"CMP      $tmp, 0 \n\t"
+"B.ne     loop \n\t" %}
+ins_encode %{
+Label loop;
+__ bind(loop);
+__ ldrexd($res$$Register, $mem$$Address);
+__ strexd($tmp$$Register, $newval$$Register, $mem$$Address);
+__ cmp($tmp$$Register, 0);
+__ b(loop, ne);
+%}
+ins_pipe( long_memory_op );
+%}
+#endif // !AARCH64
+#ifdef AARCH64
+instruct xchgP(memoryex mem, iRegP newval, iRegP res, iRegI tmp) %{
+match(Set res (GetAndSetP mem newval));
+effect(TEMP tmp, TEMP res);
+size(12);
+format %{ "loop:\n\t"
+"LDREX    $res, $mem\n\t"
+"STREX    $tmp, $newval, $mem\n\t"
+"CBNZ_w   $tmp, loop" %}
+ins_encode %{
+Label loop;
+Register base = reg_to_register_object($mem$$base);
+__ bind(loop);
+__ ldrex($res$$Register, base);
+__ strex($tmp$$Register, $newval$$Register, base);
+__ cbnz_w($tmp$$Register, loop);
+%}
+ins_pipe( long_memory_op );
+%}
+#else
+instruct xchgP(memoryex mem, iRegP newval, iRegP res, iRegI tmp, flagsReg ccr) %{
+match(Set res (GetAndSetP mem newval));
+effect(KILL ccr, TEMP tmp, TEMP res);
+size(16);
+format %{ "loop: \n\t"
+"LDREX    $res, $mem\n\t"
+"STREX    $tmp, $newval, $mem\n\t"
+"CMP      $tmp, 0 \n\t"
+"B.ne     loop \n\t" %}
+ins_encode %{
+Label loop;
+__ bind(loop);
+__ ldrex($res$$Register,$mem$$Address);
+__ strex($tmp$$Register, $newval$$Register, $mem$$Address);
+__ cmp($tmp$$Register, 0);
+__ b(loop, ne);
+%}
+ins_pipe( long_memory_op );
+%}
+#endif // !AARCH64
+//---------------------
+// Subtraction Instructions
+// Register Subtraction
+instruct subI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
+match(Set dst (SubI src1 src2));
+size(4);
+format %{ "sub_32 $dst,$src1,$src2\t! int" %}
+ins_encode %{
+__ sub_32($dst$$Register, $src1$$Register, $src2$$Register);
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#ifndef AARCH64
+instruct subshlI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
+match(Set dst (SubI src1 (LShiftI src2 src3)));
+size(4);
+format %{ "SUB    $dst,$src1,$src2<<$src3" %}
+ins_encode %{
+__ sub($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$Register));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#endif
+instruct subshlI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
+match(Set dst (SubI src1 (LShiftI src2 src3)));
+size(4);
+format %{ "sub_32 $dst,$src1,$src2<<$src3\t! int" %}
+ins_encode %{
+__ sub_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$constant));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#ifndef AARCH64
+instruct subsarI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
+match(Set dst (SubI src1 (RShiftI src2 src3)));
+size(4);
+format %{ "SUB    $dst,$src1,$src2>>$src3" %}
+ins_encode %{
+__ sub($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$Register));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#endif
+instruct subsarI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
+match(Set dst (SubI src1 (RShiftI src2 src3)));
+size(4);
+format %{ "sub_32 $dst,$src1,$src2>>$src3\t! int" %}
+ins_encode %{
+__ sub_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$constant));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#ifndef AARCH64
+instruct subshrI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
+match(Set dst (SubI src1 (URShiftI src2 src3)));
+size(4);
+format %{ "SUB    $dst,$src1,$src2>>>$src3" %}
+ins_encode %{
+__ sub($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$Register));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#endif
+instruct subshrI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
+match(Set dst (SubI src1 (URShiftI src2 src3)));
+size(4);
+format %{ "sub_32 $dst,$src1,$src2>>>$src3\t! int" %}
+ins_encode %{
+__ sub_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$constant));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#ifndef AARCH64
+instruct rsbshlI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
+match(Set dst (SubI (LShiftI src1 src2) src3));
+size(4);
+format %{ "RSB    $dst,$src3,$src1<<$src2" %}
+ins_encode %{
+__ rsb($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsl, $src2$$Register));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+instruct rsbshlI_reg_imm_reg(iRegI dst, iRegI src1, immU5 src2, iRegI src3) %{
+match(Set dst (SubI (LShiftI src1 src2) src3));
+size(4);
+format %{ "RSB    $dst,$src3,$src1<<$src2" %}
+ins_encode %{
+__ rsb($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsl, $src2$$constant));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+instruct rsbsarI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
+match(Set dst (SubI (RShiftI src1 src2) src3));
+size(4);
+format %{ "RSB    $dst,$src3,$src1>>$src2" %}
+ins_encode %{
+__ rsb($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, asr, $src2$$Register));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+instruct rsbsarI_reg_imm_reg(iRegI dst, iRegI src1, immU5 src2, iRegI src3) %{
+match(Set dst (SubI (RShiftI src1 src2) src3));
+size(4);
+format %{ "RSB    $dst,$src3,$src1>>$src2" %}
+ins_encode %{
+__ rsb($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, asr, $src2$$constant));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+instruct rsbshrI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
+match(Set dst (SubI (URShiftI src1 src2) src3));
+size(4);
+format %{ "RSB    $dst,$src3,$src1>>>$src2" %}
+ins_encode %{
+__ rsb($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsr, $src2$$Register));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+instruct rsbshrI_reg_imm_reg(iRegI dst, iRegI src1, immU5 src2, iRegI src3) %{
+match(Set dst (SubI (URShiftI src1 src2) src3));
+size(4);
+format %{ "RSB    $dst,$src3,$src1>>>$src2" %}
+ins_encode %{
+__ rsb($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsr, $src2$$constant));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#endif
+// Immediate Subtraction
+instruct subI_reg_aimmI(iRegI dst, iRegI src1, aimmI src2) %{
+match(Set dst (SubI src1 src2));
+size(4);
+format %{ "sub_32 $dst,$src1,$src2\t! int" %}
+ins_encode %{
+__ sub_32($dst$$Register, $src1$$Register, $src2$$constant);
+%}
+ins_pipe(ialu_reg_imm);
+%}
+instruct subI_reg_immRotneg(iRegI dst, iRegI src1, aimmIneg src2) %{
+match(Set dst (AddI src1 src2));
+size(4);
+format %{ "sub_32 $dst,$src1,-($src2)\t! int" %}
+ins_encode %{
+__ sub_32($dst$$Register, $src1$$Register, -$src2$$constant);
+%}
+ins_pipe(ialu_reg_imm);
+%}
+#ifndef AARCH64
+instruct subI_immRot_reg(iRegI dst, immIRot src1, iRegI src2) %{
+match(Set dst (SubI src1 src2));
+size(4);
+format %{ "RSB    $dst,$src2,src1" %}
+ins_encode %{
+__ rsb($dst$$Register, $src2$$Register, $src1$$constant);
+%}
+ins_pipe(ialu_zero_reg);
+%}
+#endif
+// Register Subtraction
+#ifdef AARCH64
+instruct subL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
+match(Set dst (SubL src1 src2));
+size(4);
+format %{ "SUB    $dst,$src1,$src2\t! long" %}
+ins_encode %{
+__ sub($dst$$Register, $src1$$Register, $src2$$Register);
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#else
+instruct subL_reg_reg(iRegL dst, iRegL src1, iRegL src2, flagsReg icc ) %{
+match(Set dst (SubL src1 src2));
+effect (KILL icc);
+size(8);
+format %{ "SUBS   $dst.lo,$src1.lo,$src2.lo\t! long\n\t"
+"SBC    $dst.hi,$src1.hi,$src2.hi" %}
+ins_encode %{
+__ subs($dst$$Register, $src1$$Register, $src2$$Register);
+__ sbc($dst$$Register->successor(), $src1$$Register->successor(), $src2$$Register->successor());
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#endif
+#ifdef AARCH64
+// Immediate Subtraction
+instruct subL_reg_aimm(iRegL dst, iRegL src1, aimmL src2) %{
+match(Set dst (SubL src1 src2));
+size(4);
+format %{ "SUB    $dst,$src1,$src2\t! long" %}
+ins_encode %{
+__ sub($dst$$Register, $src1$$Register, $src2$$constant);
+%}
+ins_pipe(ialu_reg_imm);
+%}
+instruct subL_reg_immLneg(iRegL dst, iRegL src1, aimmLneg src2) %{
+match(Set dst (AddL src1 src2));
+size(4);
+format %{ "SUB    $dst,$src1,-($src2)\t! long" %}
+ins_encode %{
+__ sub($dst$$Register, $src1$$Register, -$src2$$constant);
+%}
+ins_pipe(ialu_reg_imm);
+%}
+#else
+// TODO
+#endif
+#ifndef AARCH64
+// Immediate Subtraction
+// TODO: try immLRot2 instead, (0, $con$$constant) becomes
+// (hi($con$$constant), lo($con$$constant)) becomes
+instruct subL_reg_immRot(iRegL dst, iRegL src1, immLlowRot con, flagsReg icc) %{
+match(Set dst (SubL src1 con));
+effect (KILL icc);
+size(8);
+format %{ "SUB    $dst.lo,$src1.lo,$con\t! long\n\t"
+"SBC    $dst.hi,$src1.hi,0" %}
+ins_encode %{
+__ subs($dst$$Register, $src1$$Register, $con$$constant);
+__ sbc($dst$$Register->successor(), $src1$$Register->successor(), 0);
+%}
+ins_pipe(ialu_reg_imm);
+%}
+// Long negation
+instruct negL_reg_reg(iRegL dst, immL0 zero, iRegL src2, flagsReg icc) %{
+match(Set dst (SubL zero src2));
+effect (KILL icc);
+size(8);
+format %{ "RSBS   $dst.lo,$src2.lo,0\t! long\n\t"
+"RSC    $dst.hi,$src2.hi,0" %}
+ins_encode %{
+__ rsbs($dst$$Register, $src2$$Register, 0);
+__ rsc($dst$$Register->successor(), $src2$$Register->successor(), 0);
+%}
+ins_pipe(ialu_zero_reg);
+%}
+#endif // !AARCH64
+// Multiplication Instructions
+// Integer Multiplication
+// Register Multiplication
+instruct mulI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
+match(Set dst (MulI src1 src2));
+size(4);
+format %{ "mul_32 $dst,$src1,$src2" %}
+ins_encode %{
+__ mul_32($dst$$Register, $src1$$Register, $src2$$Register);
+%}
+ins_pipe(imul_reg_reg);
+%}
+#ifdef AARCH64
+instruct mulL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
+match(Set dst (MulL src1 src2));
+size(4);
+format %{ "MUL  $dst,$src1,$src2\t! long" %}
+ins_encode %{
+__ mul($dst$$Register, $src1$$Register, $src2$$Register);
+%}
+ins_pipe(imul_reg_reg);
+%}
+#else
+instruct mulL_lo1_hi2(iRegL dst, iRegL src1, iRegL src2) %{
+effect(DEF dst, USE src1, USE src2);
+size(4);
+format %{ "MUL  $dst.hi,$src1.lo,$src2.hi\t! long" %}
+ins_encode %{
+__ mul($dst$$Register->successor(), $src1$$Register, $src2$$Register->successor());
+%}
+ins_pipe(imul_reg_reg);
+%}
+instruct mulL_hi1_lo2(iRegL dst, iRegL src1, iRegL src2) %{
+effect(USE_DEF dst, USE src1, USE src2);
+size(8);
+format %{ "MLA  $dst.hi,$src1.hi,$src2.lo,$dst.hi\t! long\n\t"
+"MOV  $dst.lo, 0"%}
+ins_encode %{
+__ mla($dst$$Register->successor(), $src1$$Register->successor(), $src2$$Register, $dst$$Register->successor());
+__ mov($dst$$Register, 0);
+%}
+ins_pipe(imul_reg_reg);
+%}
+instruct mulL_lo1_lo2(iRegL dst, iRegL src1, iRegL src2) %{
+effect(USE_DEF dst, USE src1, USE src2);
+size(4);
+format %{ "UMLAL  $dst.lo,$dst.hi,$src1,$src2\t! long" %}
+ins_encode %{
+__ umlal($dst$$Register, $dst$$Register->successor(), $src1$$Register, $src2$$Register);
+%}
+ins_pipe(imul_reg_reg);
+%}
+instruct mulL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
+match(Set dst (MulL src1 src2));
+expand %{
+mulL_lo1_hi2(dst, src1, src2);
+mulL_hi1_lo2(dst, src1, src2);
+mulL_lo1_lo2(dst, src1, src2);
+%}
+%}
+#endif // !AARCH64
+// Integer Division
+// Register Division
+#ifdef AARCH64
+instruct divI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
+match(Set dst (DivI src1 src2));
+size(4);
+format %{ "SDIV    $dst,$src1,$src2\t! 32-bit" %}
+ins_encode %{
+__ sdiv_w($dst$$Register, $src1$$Register, $src2$$Register);
+%}
+ins_pipe(ialu_reg_reg); // FIXME
+%}
+#else
+instruct divI_reg_reg(R1RegI dst, R0RegI src1, R2RegI src2, LRRegP lr, flagsReg ccr) %{
+match(Set dst (DivI src1 src2));
+effect( KILL ccr, KILL src1, KILL src2, KILL lr);
+ins_cost((2+71)*DEFAULT_COST);
+format %{ "DIV   $dst,$src1,$src2 ! call to StubRoutines::Arm::idiv_irem_entry()" %}
+ins_encode %{
+__ call(StubRoutines::Arm::idiv_irem_entry(), relocInfo::runtime_call_type);
+%}
+ins_pipe(sdiv_reg_reg);
+%}
+#endif
+// Register Long Division
+#ifdef AARCH64
+instruct divL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
+match(Set dst (DivL src1 src2));
+size(4);
+format %{ "SDIV    $dst,$src1,$src2" %}
+ins_encode %{
+__ sdiv($dst$$Register, $src1$$Register, $src2$$Register);
+%}
+ins_pipe(ialu_reg_reg); // FIXME
+%}
+#else
+instruct divL_reg_reg(R0R1RegL dst, R2R3RegL src1, R0R1RegL src2) %{
+match(Set dst (DivL src1 src2));
+effect(CALL);
+ins_cost(DEFAULT_COST*71);
+format %{ "DIVL  $src1,$src2,$dst\t! long ! call to SharedRuntime::ldiv" %}
+ins_encode %{
+address target = CAST_FROM_FN_PTR(address, SharedRuntime::ldiv);
+__ call(target, relocInfo::runtime_call_type);
+%}
+ins_pipe(divL_reg_reg);
+%}
+#endif
+// Integer Remainder
+// Register Remainder
+#ifdef AARCH64
+#ifdef TODO
+instruct msubI_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
+match(Set dst (SubI src1 (MulI src2 src3)));
+size(4);
+format %{ "MSUB    $dst,$src2,$src3,$src1\t! 32-bit\n\t" %}
+ins_encode %{
+__ msub_w($dst$$Register, $src2$$Register, $src3$$Register, $src1$$Register);
+%}
+ins_pipe(ialu_reg_reg); // FIXME
+%}
+#endif
+instruct modI_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI temp) %{
+match(Set dst (ModI src1 src2));
+effect(TEMP temp);
+size(8);
+format %{ "SDIV    $temp,$src1,$src2\t! 32-bit\n\t"
+"MSUB    $dst,$src2,$temp,$src1\t! 32-bit\n\t" %}
+ins_encode %{
+__ sdiv_w($temp$$Register, $src1$$Register, $src2$$Register);
+__ msub_w($dst$$Register, $src2$$Register, $temp$$Register, $src1$$Register);
+%}
+ins_pipe(ialu_reg_reg); // FIXME
+%}
+#else
+instruct modI_reg_reg(R0RegI dst, R0RegI src1, R2RegI src2, R1RegI temp, LRRegP lr, flagsReg ccr ) %{
+match(Set dst (ModI src1 src2));
+effect( KILL ccr, KILL temp, KILL src2, KILL lr);
+format %{ "MODI   $dst,$src1,$src2\t ! call to StubRoutines::Arm::idiv_irem_entry" %}
+ins_encode %{
+__ call(StubRoutines::Arm::idiv_irem_entry(), relocInfo::runtime_call_type);
+%}
+ins_pipe(sdiv_reg_reg);
+%}
+#endif
+// Register Long Remainder
+#ifdef AARCH64
+instruct modL_reg_reg(iRegL dst, iRegL src1, iRegL src2, iRegL temp) %{
+match(Set dst (ModL src1 src2));
+effect(TEMP temp);
+size(8);
+format %{ "SDIV    $temp,$src1,$src2\n\t"
+"MSUB    $dst,$src2,$temp,$src1" %}
+ins_encode %{
+__ sdiv($temp$$Register, $src1$$Register, $src2$$Register);
+__ msub($dst$$Register, $src2$$Register, $temp$$Register, $src1$$Register);
+%}
+ins_pipe(ialu_reg_reg); // FIXME
+%}
+#else
+instruct modL_reg_reg(R0R1RegL dst, R2R3RegL src1, R0R1RegL src2) %{
+match(Set dst (ModL src1 src2));
+effect(CALL);
+ins_cost(MEMORY_REF_COST); // FIXME
+format %{ "modL    $dst,$src1,$src2\t ! call to SharedRuntime::lrem" %}
+ins_encode %{
+address target = CAST_FROM_FN_PTR(address, SharedRuntime::lrem);
+__ call(target, relocInfo::runtime_call_type);
+%}
+ins_pipe(divL_reg_reg);
+%}
+#endif
+// Integer Shift Instructions
+// Register Shift Left
+instruct shlI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
+match(Set dst (LShiftI src1 src2));
+size(4);
+#ifdef AARCH64
+format %{ "LSLV   $dst,$src1,$src2\t! int" %}
+ins_encode %{
+__ lslv_w($dst$$Register, $src1$$Register, $src2$$Register);
+%}
+#else
+format %{ "LSL  $dst,$src1,$src2 \n\t" %}
+ins_encode %{
+__ mov($dst$$Register, AsmOperand($src1$$Register, lsl, $src2$$Register));
+%}
+#endif
+ins_pipe(ialu_reg_reg);
+%}
+// Register Shift Left Immediate
+instruct shlI_reg_imm5(iRegI dst, iRegI src1, immU5 src2) %{
+match(Set dst (LShiftI src1 src2));
+size(4);
+#ifdef AARCH64
+format %{ "LSL_w  $dst,$src1,$src2\t! int" %}
+ins_encode %{
+__ _lsl($dst$$Register, $src1$$Register, $src2$$constant);
+%}
+#else
+format %{ "LSL    $dst,$src1,$src2\t! int" %}
+ins_encode %{
+__ logical_shift_left($dst$$Register, $src1$$Register, $src2$$constant);
+%}
+#endif
+ins_pipe(ialu_reg_imm);
+%}
+#ifndef AARCH64
+instruct shlL_reg_reg_merge_hi(iRegL dst, iRegL src1, iRegI src2) %{
+effect(USE_DEF dst, USE src1, USE src2);
+size(4);
+format %{"OR  $dst.hi,$dst.hi,($src1.hi << $src2)"  %}
+ins_encode %{
+__ orr($dst$$Register->successor(), $dst$$Register->successor(), AsmOperand($src1$$Register->successor(), lsl, $src2$$Register));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+instruct shlL_reg_reg_merge_lo(iRegL dst, iRegL src1, iRegI src2) %{
+effect(USE_DEF dst, USE src1, USE src2);
+size(4);
+format %{ "LSL  $dst.lo,$src1.lo,$src2 \n\t" %}
+ins_encode %{
+__ mov($dst$$Register, AsmOperand($src1$$Register, lsl, $src2$$Register));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+instruct shlL_reg_reg_overlap(iRegL dst, iRegL src1, iRegI src2, flagsReg ccr) %{
+effect(DEF dst, USE src1, USE src2, KILL ccr);
+size(16);
+format %{ "SUBS  $dst.hi,$src2,32 \n\t"
+"LSLpl $dst.hi,$src1.lo,$dst.hi \n\t"
+"RSBmi $dst.hi,$dst.hi,0 \n\t"
+"LSRmi $dst.hi,$src1.lo,$dst.hi" %}
+ins_encode %{
+// $src1$$Register and $dst$$Register->successor() can't be the same
+__ subs($dst$$Register->successor(), $src2$$Register, 32);
+__ mov($dst$$Register->successor(), AsmOperand($src1$$Register, lsl, $dst$$Register->successor()), pl);
+__ rsb($dst$$Register->successor(), $dst$$Register->successor(), 0, mi);
+__ mov($dst$$Register->successor(), AsmOperand($src1$$Register, lsr, $dst$$Register->successor()), mi);
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#endif // !AARCH64
+instruct shlL_reg_reg(iRegL dst, iRegL src1, iRegI src2) %{
+match(Set dst (LShiftL src1 src2));
+#ifdef AARCH64
+size(4);
+format %{ "LSLV  $dst,$src1,$src2\t! long" %}
+ins_encode %{
+__ lslv($dst$$Register, $src1$$Register, $src2$$Register);
+%}
+ins_pipe(ialu_reg_reg);
+#else
+expand %{
+flagsReg ccr;
+shlL_reg_reg_overlap(dst, src1, src2, ccr);
+shlL_reg_reg_merge_hi(dst, src1, src2);
+shlL_reg_reg_merge_lo(dst, src1, src2);
+%}
+#endif
+%}
+#ifdef AARCH64
+instruct shlL_reg_imm6(iRegL dst, iRegL src1, immU6 src2) %{
+match(Set dst (LShiftL src1 src2));
+size(4);
+format %{ "LSL    $dst,$src1,$src2\t! long" %}
+ins_encode %{
+__ logical_shift_left($dst$$Register, $src1$$Register, $src2$$constant);
+%}
+ins_pipe(ialu_reg_imm);
+%}
+#else
+// Register Shift Left Immediate
+instruct shlL_reg_imm6(iRegL dst, iRegL src1, immU6Big src2) %{
+match(Set dst (LShiftL src1 src2));
+size(8);
+format %{ "LSL   $dst.hi,$src1.lo,$src2-32\t! or mov if $src2==32\n\t"
+"MOV   $dst.lo, 0" %}
+ins_encode %{
+if ($src2$$constant == 32) {
+__ mov($dst$$Register->successor(), $src1$$Register);
+} else {
+__ mov($dst$$Register->successor(), AsmOperand($src1$$Register, lsl, $src2$$constant-32));
+}
+__ mov($dst$$Register, 0);
+%}
+ins_pipe(ialu_reg_imm);
+%}
+instruct shlL_reg_imm5(iRegL dst, iRegL src1, immU5 src2) %{
+match(Set dst (LShiftL src1 src2));
+size(12);
+format %{ "LSL   $dst.hi,$src1.lo,$src2\n\t"
+"OR    $dst.hi, $dst.hi, $src1.lo >> 32-$src2\n\t"
+"LSL   $dst.lo,$src1.lo,$src2" %}
+ins_encode %{
+// The order of the following 3 instructions matters: src1.lo and
+// dst.hi can't overlap but src.hi and dst.hi can.
+__ mov($dst$$Register->successor(), AsmOperand($src1$$Register->successor(), lsl, $src2$$constant));
+__ orr($dst$$Register->successor(), $dst$$Register->successor(), AsmOperand($src1$$Register, lsr, 32-$src2$$constant));
+__ mov($dst$$Register, AsmOperand($src1$$Register, lsl, $src2$$constant));
+%}
+ins_pipe(ialu_reg_imm);
+%}
+#endif // !AARCH64
+// Register Arithmetic Shift Right
+instruct sarI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
+match(Set dst (RShiftI src1 src2));
+size(4);
+#ifdef AARCH64
+format %{ "ASRV   $dst,$src1,$src2\t! int" %}
+ins_encode %{
+__ asrv_w($dst$$Register, $src1$$Register, $src2$$Register);
+%}
+#else
+format %{ "ASR    $dst,$src1,$src2\t! int" %}
+ins_encode %{
+__ mov($dst$$Register, AsmOperand($src1$$Register, asr, $src2$$Register));
+%}
+#endif
+ins_pipe(ialu_reg_reg);
+%}
+// Register Arithmetic Shift Right Immediate
+instruct sarI_reg_imm5(iRegI dst, iRegI src1, immU5 src2) %{
+match(Set dst (RShiftI src1 src2));
+size(4);
+#ifdef AARCH64
+format %{ "ASR_w  $dst,$src1,$src2" %}
+ins_encode %{
+__ _asr_w($dst$$Register, $src1$$Register, $src2$$constant);
+%}
+#else
+format %{ "ASR    $dst,$src1,$src2" %}
+ins_encode %{
+__ mov($dst$$Register, AsmOperand($src1$$Register, asr, $src2$$constant));
+%}
+#endif
+ins_pipe(ialu_reg_imm);
+%}
+#ifndef AARCH64
+// Register Shift Right Arithmetic Long
+instruct sarL_reg_reg_merge_lo(iRegL dst, iRegL src1, iRegI src2) %{
+effect(USE_DEF dst, USE src1, USE src2);
+size(4);
+format %{ "OR  $dst.lo,$dst.lo,($src1.lo >> $src2)"  %}
+ins_encode %{
+__ orr($dst$$Register, $dst$$Register, AsmOperand($src1$$Register, lsr, $src2$$Register));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+instruct sarL_reg_reg_merge_hi(iRegL dst, iRegL src1, iRegI src2) %{
+effect(USE_DEF dst, USE src1, USE src2);
+size(4);
+format %{ "ASR  $dst.hi,$src1.hi,$src2 \n\t" %}
+ins_encode %{
+__ mov($dst$$Register->successor(), AsmOperand($src1$$Register->successor(), asr, $src2$$Register));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+instruct sarL_reg_reg_overlap(iRegL dst, iRegL src1, iRegI src2, flagsReg ccr) %{
+effect(DEF dst, USE src1, USE src2, KILL ccr);
+size(16);
+format %{ "SUBS  $dst.lo,$src2,32 \n\t"
+"ASRpl $dst.lo,$src1.hi,$dst.lo \n\t"
+"RSBmi $dst.lo,$dst.lo,0 \n\t"
+"LSLmi $dst.lo,$src1.hi,$dst.lo" %}
+ins_encode %{
+// $src1$$Register->successor() and $dst$$Register can't be the same
+__ subs($dst$$Register, $src2$$Register, 32);
+__ mov($dst$$Register, AsmOperand($src1$$Register->successor(), asr, $dst$$Register), pl);
+__ rsb($dst$$Register, $dst$$Register, 0, mi);
+__ mov($dst$$Register, AsmOperand($src1$$Register->successor(), lsl, $dst$$Register), mi);
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#endif // !AARCH64
+instruct sarL_reg_reg(iRegL dst, iRegL src1, iRegI src2) %{
+match(Set dst (RShiftL src1 src2));
+#ifdef AARCH64
+size(4);
+format %{ "ASRV  $dst,$src1,$src2\t! long" %}
+ins_encode %{
+__ asrv($dst$$Register, $src1$$Register, $src2$$Register);
+%}
+ins_pipe(ialu_reg_reg);
+#else
+expand %{
+flagsReg ccr;
+sarL_reg_reg_overlap(dst, src1, src2, ccr);
+sarL_reg_reg_merge_lo(dst, src1, src2);
+sarL_reg_reg_merge_hi(dst, src1, src2);
+%}
+#endif
+%}
+// Register Shift Left Immediate
+#ifdef AARCH64
+instruct sarL_reg_imm6(iRegL dst, iRegL src1, immU6 src2) %{
+match(Set dst (RShiftL src1 src2));
+size(4);
+format %{ "ASR    $dst,$src1,$src2\t! long" %}
+ins_encode %{
+__ _asr($dst$$Register, $src1$$Register, $src2$$constant);
+%}
+ins_pipe(ialu_reg_imm);
+%}
+#else
+instruct sarL_reg_imm6(iRegL dst, iRegL src1, immU6Big src2) %{
+match(Set dst (RShiftL src1 src2));
+size(8);
+format %{ "ASR   $dst.lo,$src1.hi,$src2-32\t! or mov if $src2==32\n\t"
+"ASR   $dst.hi,$src1.hi, $src2" %}
+ins_encode %{
+if ($src2$$constant == 32) {
+__ mov($dst$$Register, $src1$$Register->successor());
+} else{
+__ mov($dst$$Register, AsmOperand($src1$$Register->successor(), asr, $src2$$constant-32));
+}
+__ mov($dst$$Register->successor(), AsmOperand($src1$$Register->successor(), asr, 0));
+%}
+ins_pipe(ialu_reg_imm);
+%}
+instruct sarL_reg_imm5(iRegL dst, iRegL src1, immU5 src2) %{
+match(Set dst (RShiftL src1 src2));
+size(12);
+format %{ "LSR   $dst.lo,$src1.lo,$src2\n\t"
+"OR    $dst.lo, $dst.lo, $src1.hi << 32-$src2\n\t"
+"ASR   $dst.hi,$src1.hi,$src2" %}
+ins_encode %{
+// The order of the following 3 instructions matters: src1.lo and
+// dst.hi can't overlap but src.hi and dst.hi can.
+__ mov($dst$$Register, AsmOperand($src1$$Register, lsr, $src2$$constant));
+__ orr($dst$$Register, $dst$$Register, AsmOperand($src1$$Register->successor(), lsl, 32-$src2$$constant));
+__ mov($dst$$Register->successor(), AsmOperand($src1$$Register->successor(), asr, $src2$$constant));
+%}
+ins_pipe(ialu_reg_imm);
+%}
+#endif
+// Register Shift Right
+instruct shrI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
+match(Set dst (URShiftI src1 src2));
+size(4);
+#ifdef AARCH64
+format %{ "LSRV   $dst,$src1,$src2\t! int" %}
+ins_encode %{
+__ lsrv_w($dst$$Register, $src1$$Register, $src2$$Register);
+%}
+#else
+format %{ "LSR    $dst,$src1,$src2\t! int" %}
+ins_encode %{
+__ mov($dst$$Register, AsmOperand($src1$$Register, lsr, $src2$$Register));
+%}
+#endif
+ins_pipe(ialu_reg_reg);
+%}
+// Register Shift Right Immediate
+instruct shrI_reg_imm5(iRegI dst, iRegI src1, immU5 src2) %{
+match(Set dst (URShiftI src1 src2));
+size(4);
+#ifdef AARCH64
+format %{ "LSR_w  $dst,$src1,$src2" %}
+ins_encode %{
+__ _lsr_w($dst$$Register, $src1$$Register, $src2$$constant);
+%}
+#else
+format %{ "LSR    $dst,$src1,$src2" %}
+ins_encode %{
+__ mov($dst$$Register, AsmOperand($src1$$Register, lsr, $src2$$constant));
+%}
+#endif
+ins_pipe(ialu_reg_imm);
+%}
+#ifndef AARCH64
+// Register Shift Right
+instruct shrL_reg_reg_merge_lo(iRegL dst, iRegL src1, iRegI src2) %{
+effect(USE_DEF dst, USE src1, USE src2);
+size(4);
+format %{ "OR   $dst.lo,$dst,($src1.lo >>> $src2)"  %}
+ins_encode %{
+__ orr($dst$$Register, $dst$$Register, AsmOperand($src1$$Register, lsr, $src2$$Register));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+instruct shrL_reg_reg_merge_hi(iRegL dst, iRegL src1, iRegI src2) %{
+effect(USE_DEF dst, USE src1, USE src2);
+size(4);
+format %{ "LSR  $dst.hi,$src1.hi,$src2 \n\t" %}
+ins_encode %{
+__ mov($dst$$Register->successor(), AsmOperand($src1$$Register->successor(), lsr, $src2$$Register));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+instruct shrL_reg_reg_overlap(iRegL dst, iRegL src1, iRegI src2, flagsReg ccr) %{
+effect(DEF dst, USE src1, USE src2, KILL ccr);
+size(16);
+format %{ "SUBS  $dst,$src2,32 \n\t"
+"LSRpl $dst,$src1.hi,$dst \n\t"
+"RSBmi $dst,$dst,0 \n\t"
+"LSLmi $dst,$src1.hi,$dst" %}
+ins_encode %{
+// $src1$$Register->successor() and $dst$$Register can't be the same
+__ subs($dst$$Register, $src2$$Register, 32);
+__ mov($dst$$Register, AsmOperand($src1$$Register->successor(), lsr, $dst$$Register), pl);
+__ rsb($dst$$Register, $dst$$Register, 0, mi);
+__ mov($dst$$Register, AsmOperand($src1$$Register->successor(), lsl, $dst$$Register), mi);
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#endif // !AARCH64
+instruct shrL_reg_reg(iRegL dst, iRegL src1, iRegI src2) %{
+match(Set dst (URShiftL src1 src2));
+#ifdef AARCH64
+size(4);
+format %{ "LSRV  $dst,$src1,$src2\t! long" %}
+ins_encode %{
+__ lsrv($dst$$Register, $src1$$Register, $src2$$Register);
+%}
+ins_pipe(ialu_reg_reg);
+#else
+expand %{
+flagsReg ccr;
+shrL_reg_reg_overlap(dst, src1, src2, ccr);
+shrL_reg_reg_merge_lo(dst, src1, src2);
+shrL_reg_reg_merge_hi(dst, src1, src2);
+%}
+#endif
+%}
+// Register Shift Right Immediate
+#ifdef AARCH64
+instruct shrL_reg_imm6(iRegL dst, iRegL src1, immU6 src2) %{
+match(Set dst (URShiftL src1 src2));
+size(4);
+format %{ "LSR    $dst,$src1,$src2" %}
+ins_encode %{
+__ _lsr($dst$$Register, $src1$$Register, $src2$$constant);
+%}
+ins_pipe(ialu_reg_imm);
+%}
+#else
+instruct shrL_reg_imm6(iRegL dst, iRegL src1, immU6Big src2) %{
+match(Set dst (URShiftL src1 src2));
+size(8);
+format %{ "LSR   $dst.lo,$src1.hi,$src2-32\t! or mov if $src2==32\n\t"
+"MOV   $dst.hi, 0" %}
+ins_encode %{
+if ($src2$$constant == 32) {
+__ mov($dst$$Register, $src1$$Register->successor());
+} else {
+__ mov($dst$$Register, AsmOperand($src1$$Register->successor(), lsr, $src2$$constant-32));
+}
+__ mov($dst$$Register->successor(), 0);
+%}
+ins_pipe(ialu_reg_imm);
+%}
+instruct shrL_reg_imm5(iRegL dst, iRegL src1, immU5 src2) %{
+match(Set dst (URShiftL src1 src2));
+size(12);
+format %{ "LSR   $dst.lo,$src1.lo,$src2\n\t"
+"OR    $dst.lo, $dst.lo, $src1.hi << 32-$src2\n\t"
+"LSR   $dst.hi,$src1.hi,$src2" %}
+ins_encode %{
+// The order of the following 3 instructions matters: src1.lo and
+// dst.hi can't overlap but src.hi and dst.hi can.
+__ mov($dst$$Register, AsmOperand($src1$$Register, lsr, $src2$$constant));
+__ orr($dst$$Register, $dst$$Register, AsmOperand($src1$$Register->successor(), lsl, 32-$src2$$constant));
+__ mov($dst$$Register->successor(), AsmOperand($src1$$Register->successor(), lsr, $src2$$constant));
+%}
+ins_pipe(ialu_reg_imm);
+%}
+#endif // !AARCH64
+instruct shrP_reg_imm5(iRegX dst, iRegP src1, immU5 src2) %{
+match(Set dst (URShiftI (CastP2X src1) src2));
+size(4);
+format %{ "LSR    $dst,$src1,$src2\t! Cast ptr $src1 to int and shift" %}
+ins_encode %{
+__ logical_shift_right($dst$$Register, $src1$$Register, $src2$$constant);
+%}
+ins_pipe(ialu_reg_imm);
+%}
+//----------Floating Point Arithmetic Instructions-----------------------------
+//  Add float single precision
+instruct addF_reg_reg(regF dst, regF src1, regF src2) %{
+match(Set dst (AddF src1 src2));
+size(4);
+format %{ "FADDS  $dst,$src1,$src2" %}
+ins_encode %{
+__ add_float($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
+%}
+ins_pipe(faddF_reg_reg);
+%}
+//  Add float double precision
+instruct addD_reg_reg(regD dst, regD src1, regD src2) %{
+match(Set dst (AddD src1 src2));
+size(4);
+format %{ "FADDD  $dst,$src1,$src2" %}
+ins_encode %{
+__ add_double($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
+%}
+ins_pipe(faddD_reg_reg);
+%}
+//  Sub float single precision
+instruct subF_reg_reg(regF dst, regF src1, regF src2) %{
+match(Set dst (SubF src1 src2));
+size(4);
+format %{ "FSUBS  $dst,$src1,$src2" %}
+ins_encode %{
+__ sub_float($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
+%}
+ins_pipe(faddF_reg_reg);
+%}
+//  Sub float double precision
+instruct subD_reg_reg(regD dst, regD src1, regD src2) %{
+match(Set dst (SubD src1 src2));
+size(4);
+format %{ "FSUBD  $dst,$src1,$src2" %}
+ins_encode %{
+__ sub_double($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
+%}
+ins_pipe(faddD_reg_reg);
+%}
+//  Mul float single precision
+instruct mulF_reg_reg(regF dst, regF src1, regF src2) %{
+match(Set dst (MulF src1 src2));
+size(4);
+format %{ "FMULS  $dst,$src1,$src2" %}
+ins_encode %{
+__ mul_float($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
+%}
+ins_pipe(fmulF_reg_reg);
+%}
+//  Mul float double precision
+instruct mulD_reg_reg(regD dst, regD src1, regD src2) %{
+match(Set dst (MulD src1 src2));
+size(4);
+format %{ "FMULD  $dst,$src1,$src2" %}
+ins_encode %{
+__ mul_double($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
+%}
+ins_pipe(fmulD_reg_reg);
+%}
+//  Div float single precision
+instruct divF_reg_reg(regF dst, regF src1, regF src2) %{
+match(Set dst (DivF src1 src2));
+size(4);
+format %{ "FDIVS  $dst,$src1,$src2" %}
+ins_encode %{
+__ div_float($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
+%}
+ins_pipe(fdivF_reg_reg);
+%}
+//  Div float double precision
+instruct divD_reg_reg(regD dst, regD src1, regD src2) %{
+match(Set dst (DivD src1 src2));
+size(4);
+format %{ "FDIVD  $dst,$src1,$src2" %}
+ins_encode %{
+__ div_double($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
+%}
+ins_pipe(fdivD_reg_reg);
+%}
+//  Absolute float double precision
+instruct absD_reg(regD dst, regD src) %{
+match(Set dst (AbsD src));
+size(4);
+format %{ "FABSd  $dst,$src" %}
+ins_encode %{
+__ abs_double($dst$$FloatRegister, $src$$FloatRegister);
+%}
+ins_pipe(faddD_reg);
+%}
+//  Absolute float single precision
+instruct absF_reg(regF dst, regF src) %{
+match(Set dst (AbsF src));
+format %{ "FABSs  $dst,$src" %}
+ins_encode %{
+__ abs_float($dst$$FloatRegister, $src$$FloatRegister);
+%}
+ins_pipe(faddF_reg);
+%}
+instruct negF_reg(regF dst, regF src) %{
+match(Set dst (NegF src));
+size(4);
+format %{ "FNEGs  $dst,$src" %}
+ins_encode %{
+__ neg_float($dst$$FloatRegister, $src$$FloatRegister);
+%}
+ins_pipe(faddF_reg);
+%}
+instruct negD_reg(regD dst, regD src) %{
+match(Set dst (NegD src));
+format %{ "FNEGd  $dst,$src" %}
+ins_encode %{
+__ neg_double($dst$$FloatRegister, $src$$FloatRegister);
+%}
+ins_pipe(faddD_reg);
+%}
+//  Sqrt float double precision
+instruct sqrtF_reg_reg(regF dst, regF src) %{
+match(Set dst (ConvD2F (SqrtD (ConvF2D src))));
+size(4);
+format %{ "FSQRTS $dst,$src" %}
+ins_encode %{
+__ sqrt_float($dst$$FloatRegister, $src$$FloatRegister);
+%}
+ins_pipe(fdivF_reg_reg);
+%}
+//  Sqrt float double precision
+instruct sqrtD_reg_reg(regD dst, regD src) %{
+match(Set dst (SqrtD src));
+size(4);
+format %{ "FSQRTD $dst,$src" %}
+ins_encode %{
+__ sqrt_double($dst$$FloatRegister, $src$$FloatRegister);
+%}
+ins_pipe(fdivD_reg_reg);
+%}
+//----------Logical Instructions-----------------------------------------------
+// And Instructions
+// Register And
+instruct andI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
+match(Set dst (AndI src1 src2));
+size(4);
+format %{ "and_32 $dst,$src1,$src2" %}
+ins_encode %{
+__ and_32($dst$$Register, $src1$$Register, $src2$$Register);
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#ifndef AARCH64
+instruct andshlI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
+match(Set dst (AndI src1 (LShiftI src2 src3)));
+size(4);
+format %{ "AND    $dst,$src1,$src2<<$src3" %}
+ins_encode %{
+__ andr($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$Register));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#endif
+instruct andshlI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
+match(Set dst (AndI src1 (LShiftI src2 src3)));
+size(4);
+format %{ "and_32 $dst,$src1,$src2<<$src3" %}
+ins_encode %{
+__ and_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$constant));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#ifndef AARCH64
+instruct andsarI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
+match(Set dst (AndI src1 (RShiftI src2 src3)));
+size(4);
+format %{ "AND    $dst,$src1,$src2>>$src3" %}
+ins_encode %{
+__ andr($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$Register));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#endif
+instruct andsarI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
+match(Set dst (AndI src1 (RShiftI src2 src3)));
+size(4);
+format %{ "and_32 $dst,$src1,$src2>>$src3" %}
+ins_encode %{
+__ and_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$constant));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#ifndef AARCH64
+instruct andshrI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
+match(Set dst (AndI src1 (URShiftI src2 src3)));
+size(4);
+format %{ "AND    $dst,$src1,$src2>>>$src3" %}
+ins_encode %{
+__ andr($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$Register));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#endif
+instruct andshrI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
+match(Set dst (AndI src1 (URShiftI src2 src3)));
+size(4);
+format %{ "and_32 $dst,$src1,$src2>>>$src3" %}
+ins_encode %{
+__ and_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$constant));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+// Immediate And
+instruct andI_reg_limm(iRegI dst, iRegI src1, limmI src2) %{
+match(Set dst (AndI src1 src2));
+size(4);
+format %{ "and_32 $dst,$src1,$src2\t! int" %}
+ins_encode %{
+__ and_32($dst$$Register, $src1$$Register, $src2$$constant);
+%}
+ins_pipe(ialu_reg_imm);
+%}
+#ifndef AARCH64
+instruct andI_reg_limmn(iRegI dst, iRegI src1, limmIn src2) %{
+match(Set dst (AndI src1 src2));
+size(4);
+format %{ "bic    $dst,$src1,~$src2\t! int" %}
+ins_encode %{
+__ bic($dst$$Register, $src1$$Register, ~$src2$$constant);
+%}
+ins_pipe(ialu_reg_imm);
+%}
+#endif
+// Register And Long
+instruct andL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
+match(Set dst (AndL src1 src2));
+ins_cost(DEFAULT_COST);
+#ifdef AARCH64
+size(4);
+format %{ "AND    $dst,$src1,$src2\t! long" %}
+ins_encode %{
+__ andr($dst$$Register, $src1$$Register, $src2$$Register);
+%}
+#else
+size(8);
+format %{ "AND    $dst,$src1,$src2\t! long" %}
+ins_encode %{
+__ andr($dst$$Register, $src1$$Register, $src2$$Register);
+__ andr($dst$$Register->successor(), $src1$$Register->successor(), $src2$$Register->successor());
+%}
+#endif
+ins_pipe(ialu_reg_reg);
+%}
+#ifdef AARCH64
+// Immediate And
+instruct andL_reg_limm(iRegL dst, iRegL src1, limmL src2) %{
+match(Set dst (AndL src1 src2));
+size(4);
+format %{ "AND    $dst,$src1,$src2\t! long" %}
+ins_encode %{
+__ andr($dst$$Register, $src1$$Register, (uintx)$src2$$constant);
+%}
+ins_pipe(ialu_reg_imm);
+%}
+#else
+// TODO: try immLRot2 instead, (0, $con$$constant) becomes
+// (hi($con$$constant), lo($con$$constant)) becomes
+instruct andL_reg_immRot(iRegL dst, iRegL src1, immLlowRot con) %{
+match(Set dst (AndL src1 con));
+ins_cost(DEFAULT_COST);
+size(8);
+format %{ "AND    $dst,$src1,$con\t! long" %}
+ins_encode %{
+__ andr($dst$$Register, $src1$$Register, $con$$constant);
+__ andr($dst$$Register->successor(), $src1$$Register->successor(), 0);
+%}
+ins_pipe(ialu_reg_imm);
+%}
+#endif
+// Or Instructions
+// Register Or
+instruct orI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
+match(Set dst (OrI src1 src2));
+size(4);
+format %{ "orr_32 $dst,$src1,$src2\t! int" %}
+ins_encode %{
+__ orr_32($dst$$Register, $src1$$Register, $src2$$Register);
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#ifndef AARCH64
+instruct orshlI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
+match(Set dst (OrI src1 (LShiftI src2 src3)));
+size(4);
+format %{ "OR    $dst,$src1,$src2<<$src3" %}
+ins_encode %{
+__ orr($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$Register));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#endif
+instruct orshlI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
+match(Set dst (OrI src1 (LShiftI src2 src3)));
+size(4);
+format %{ "orr_32 $dst,$src1,$src2<<$src3" %}
+ins_encode %{
+__ orr_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$constant));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#ifndef AARCH64
+instruct orsarI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
+match(Set dst (OrI src1 (RShiftI src2 src3)));
+size(4);
+format %{ "OR    $dst,$src1,$src2>>$src3" %}
+ins_encode %{
+__ orr($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$Register));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#endif
+instruct orsarI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
+match(Set dst (OrI src1 (RShiftI src2 src3)));
+size(4);
+format %{ "orr_32 $dst,$src1,$src2>>$src3" %}
+ins_encode %{
+__ orr_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$constant));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#ifndef AARCH64
+instruct orshrI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
+match(Set dst (OrI src1 (URShiftI src2 src3)));
+size(4);
+format %{ "OR    $dst,$src1,$src2>>>$src3" %}
+ins_encode %{
+__ orr($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$Register));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#endif
+instruct orshrI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
+match(Set dst (OrI src1 (URShiftI src2 src3)));
+size(4);
+format %{ "orr_32 $dst,$src1,$src2>>>$src3" %}
+ins_encode %{
+__ orr_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$constant));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+// Immediate Or
+instruct orI_reg_limm(iRegI dst, iRegI src1, limmI src2) %{
+match(Set dst (OrI src1 src2));
+size(4);
+format %{ "orr_32  $dst,$src1,$src2" %}
+ins_encode %{
+__ orr_32($dst$$Register, $src1$$Register, $src2$$constant);
+%}
+ins_pipe(ialu_reg_imm);
+%}
+// TODO: orn_32 with limmIn
+// Register Or Long
+instruct orL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
+match(Set dst (OrL src1 src2));
+ins_cost(DEFAULT_COST);
+#ifdef AARCH64
+size(4);
+format %{ "OR     $dst,$src1,$src2\t! long" %}
+ins_encode %{
+__ orr($dst$$Register, $src1$$Register, $src2$$Register);
+%}
+#else
+size(8);
+format %{ "OR     $dst.lo,$src1.lo,$src2.lo\t! long\n\t"
+"OR     $dst.hi,$src1.hi,$src2.hi" %}
+ins_encode %{
+__ orr($dst$$Register, $src1$$Register, $src2$$Register);
+__ orr($dst$$Register->successor(), $src1$$Register->successor(), $src2$$Register->successor());
+%}
+#endif
+ins_pipe(ialu_reg_reg);
+%}
+#ifdef AARCH64
+instruct orL_reg_limm(iRegL dst, iRegL src1, limmL src2) %{
+match(Set dst (OrL src1 src2));
+size(4);
+format %{ "ORR    $dst,$src1,$src2\t! long" %}
+ins_encode %{
+__ orr($dst$$Register, $src1$$Register, (uintx)$src2$$constant);
+%}
+ins_pipe(ialu_reg_imm);
+%}
+#else
+// TODO: try immLRot2 instead, (0, $con$$constant) becomes
+// (hi($con$$constant), lo($con$$constant)) becomes
+instruct orL_reg_immRot(iRegL dst, iRegL src1, immLlowRot con) %{
+match(Set dst (OrL src1 con));
+ins_cost(DEFAULT_COST);
+size(8);
+format %{ "OR     $dst.lo,$src1.lo,$con\t! long\n\t"
+"OR     $dst.hi,$src1.hi,$con" %}
+ins_encode %{
+__ orr($dst$$Register, $src1$$Register, $con$$constant);
+__ orr($dst$$Register->successor(), $src1$$Register->successor(), 0);
+%}
+ins_pipe(ialu_reg_imm);
+%}
+#endif
+#ifdef TODO
+// Use SPRegP to match Rthread (TLS register) without spilling.
+// Use store_ptr_RegP to match Rthread (TLS register) without spilling.
+// Use sp_ptr_RegP to match Rthread (TLS register) without spilling.
+instruct orI_reg_castP2X(iRegI dst, iRegI src1, sp_ptr_RegP src2) %{
+match(Set dst (OrI src1 (CastP2X src2)));
+size(4);
+format %{ "OR     $dst,$src1,$src2" %}
+ins_encode %{
+__ orr($dst$$Register, $src1$$Register, $src2$$Register);
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#endif
+// Xor Instructions
+// Register Xor
+instruct xorI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
+match(Set dst (XorI src1 src2));
+size(4);
+format %{ "eor_32 $dst,$src1,$src2" %}
+ins_encode %{
+__ eor_32($dst$$Register, $src1$$Register, $src2$$Register);
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#ifndef AARCH64
+instruct xorshlI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
+match(Set dst (XorI src1 (LShiftI src2 src3)));
+size(4);
+format %{ "XOR    $dst,$src1,$src2<<$src3" %}
+ins_encode %{
+__ eor($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$Register));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#endif
+instruct xorshlI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
+match(Set dst (XorI src1 (LShiftI src2 src3)));
+size(4);
+format %{ "eor_32 $dst,$src1,$src2<<$src3" %}
+ins_encode %{
+__ eor_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$constant));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#ifndef AARCH64
+instruct xorsarI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
+match(Set dst (XorI src1 (RShiftI src2 src3)));
+size(4);
+format %{ "XOR    $dst,$src1,$src2>>$src3" %}
+ins_encode %{
+__ eor($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$Register));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#endif
+instruct xorsarI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
+match(Set dst (XorI src1 (RShiftI src2 src3)));
+size(4);
+format %{ "eor_32 $dst,$src1,$src2>>$src3" %}
+ins_encode %{
+__ eor_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$constant));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#ifndef AARCH64
+instruct xorshrI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
+match(Set dst (XorI src1 (URShiftI src2 src3)));
+size(4);
+format %{ "XOR    $dst,$src1,$src2>>>$src3" %}
+ins_encode %{
+__ eor($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$Register));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+#endif
+instruct xorshrI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
+match(Set dst (XorI src1 (URShiftI src2 src3)));
+size(4);
+format %{ "eor_32 $dst,$src1,$src2>>>$src3" %}
+ins_encode %{
+__ eor_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$constant));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+// Immediate Xor
+instruct xorI_reg_imm(iRegI dst, iRegI src1, limmI src2) %{
+match(Set dst (XorI src1 src2));
+size(4);
+format %{ "eor_32 $dst,$src1,$src2" %}
+ins_encode %{
+__ eor_32($dst$$Register, $src1$$Register, $src2$$constant);
+%}
+ins_pipe(ialu_reg_imm);
+%}
+// Register Xor Long
+instruct xorL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
+match(Set dst (XorL src1 src2));
+ins_cost(DEFAULT_COST);
+#ifdef AARCH64
+size(4);
+format %{ "XOR     $dst,$src1,$src2\t! long" %}
+ins_encode %{
+__ eor($dst$$Register, $src1$$Register, $src2$$Register);
+%}
+#else
+size(8);
+format %{ "XOR     $dst.hi,$src1.hi,$src2.hi\t! long\n\t"
+"XOR     $dst.lo,$src1.lo,$src2.lo\t! long" %}
+ins_encode %{
+__ eor($dst$$Register, $src1$$Register, $src2$$Register);
+__ eor($dst$$Register->successor(), $src1$$Register->successor(), $src2$$Register->successor());
+%}
+#endif
+ins_pipe(ialu_reg_reg);
+%}
+#ifdef AARCH64
+instruct xorL_reg_limmL(iRegL dst, iRegL src1, limmL con) %{
+match(Set dst (XorL src1 con));
+ins_cost(DEFAULT_COST);
+size(4);
+format %{ "EOR     $dst,$src1,$con\t! long" %}
+ins_encode %{
+__ eor($dst$$Register, $src1$$Register, (uintx)$con$$constant);
+%}
+ins_pipe(ialu_reg_imm);
+%}
+#else
+// TODO: try immLRot2 instead, (0, $con$$constant) becomes
+// (hi($con$$constant), lo($con$$constant)) becomes
+instruct xorL_reg_immRot(iRegL dst, iRegL src1, immLlowRot con) %{
+match(Set dst (XorL src1 con));
+ins_cost(DEFAULT_COST);
+size(8);
+format %{ "XOR     $dst.hi,$src1.hi,$con\t! long\n\t"
+"XOR     $dst.lo,$src1.lo,0\t! long" %}
+ins_encode %{
+__ eor($dst$$Register, $src1$$Register, $con$$constant);
+__ eor($dst$$Register->successor(), $src1$$Register->successor(), 0);
+%}
+ins_pipe(ialu_reg_imm);
+%}
+#endif // AARCH64
+//----------Convert to Boolean-------------------------------------------------
+instruct convI2B( iRegI dst, iRegI src, flagsReg ccr ) %{
+match(Set dst (Conv2B src));
+effect(KILL ccr);
+#ifdef AARCH64
+size(8);
+ins_cost(DEFAULT_COST*2);
+format %{ "cmp_32 $src,ZR\n\t"
+"cset_w $dst, ne" %}
+ins_encode %{
+__ cmp_32($src$$Register, ZR);
+__ cset_w($dst$$Register, ne);
+%}
+#else
+size(12);
+ins_cost(DEFAULT_COST*2);
+format %{ "TST    $src,$src \n\t"
+"MOV    $dst, 0   \n\t"
+"MOV.ne $dst, 1" %}
+ins_encode %{ // FIXME: can do better?
+__ tst($src$$Register, $src$$Register);
+__ mov($dst$$Register, 0);
+__ mov($dst$$Register, 1, ne);
+%}
+#endif
+ins_pipe(ialu_reg_ialu);
+%}
+instruct convP2B( iRegI dst, iRegP src, flagsReg ccr ) %{
+match(Set dst (Conv2B src));
+effect(KILL ccr);
+#ifdef AARCH64
+size(8);
+ins_cost(DEFAULT_COST*2);
+format %{ "CMP    $src,ZR\n\t"
+"cset   $dst, ne" %}
+ins_encode %{
+__ cmp($src$$Register, ZR);
+__ cset($dst$$Register, ne);
+%}
+#else
+size(12);
+ins_cost(DEFAULT_COST*2);
+format %{ "TST    $src,$src \n\t"
+"MOV    $dst, 0   \n\t"
+"MOV.ne $dst, 1" %}
+ins_encode %{
+__ tst($src$$Register, $src$$Register);
+__ mov($dst$$Register, 0);
+__ mov($dst$$Register, 1, ne);
+%}
+#endif
+ins_pipe(ialu_reg_ialu);
+%}
+instruct cmpLTMask_reg_reg( iRegI dst, iRegI p, iRegI q, flagsReg ccr ) %{
+match(Set dst (CmpLTMask p q));
+effect( KILL ccr );
+#ifdef AARCH64
+size(8);
+ins_cost(DEFAULT_COST*2);
+format %{ "CMP_w   $p,$q\n\t"
+"CSETM_w $dst, lt" %}
+ins_encode %{
+__ cmp_w($p$$Register, $q$$Register);
+__ csetm_w($dst$$Register, lt);
+%}
+#else
+ins_cost(DEFAULT_COST*3);
+format %{ "CMP    $p,$q\n\t"
+"MOV    $dst, #0\n\t"
+"MOV.lt $dst, #-1" %}
+ins_encode %{
+__ cmp($p$$Register, $q$$Register);
+__ mov($dst$$Register, 0);
+__ mvn($dst$$Register, 0, lt);
+%}
+#endif
+ins_pipe(ialu_reg_reg_ialu);
+%}
+instruct cmpLTMask_reg_imm( iRegI dst, iRegI p, aimmI q, flagsReg ccr ) %{
+match(Set dst (CmpLTMask p q));
+effect( KILL ccr );
+#ifdef AARCH64
+size(8);
+ins_cost(DEFAULT_COST*2);
+format %{ "CMP_w   $p,$q\n\t"
+"CSETM_w $dst, lt" %}
+ins_encode %{
+__ cmp_w($p$$Register, $q$$constant);
+__ csetm_w($dst$$Register, lt);
+%}
+#else
+ins_cost(DEFAULT_COST*3);
+format %{ "CMP    $p,$q\n\t"
+"MOV    $dst, #0\n\t"
+"MOV.lt $dst, #-1" %}
+ins_encode %{
+__ cmp($p$$Register, $q$$constant);
+__ mov($dst$$Register, 0);
+__ mvn($dst$$Register, 0, lt);
+%}
+#endif
+ins_pipe(ialu_reg_reg_ialu);
+%}
+#ifdef AARCH64
+instruct cadd_cmpLTMask3( iRegI dst, iRegI p, iRegI q, iRegI y, iRegI x, flagsReg ccr ) %{
+match(Set dst (AddI (AndI (CmpLTMask p q) y) x));
+effect( TEMP dst, KILL ccr );
+size(12);
+ins_cost(DEFAULT_COST*3);
+format %{ "CMP_w  $p,$q\n\t"
+"ADD_w  $dst,$y,$x\n\t"
+"CSEL_w $dst,$dst,$x,lt" %}
+ins_encode %{
+__ cmp_w($p$$Register, $q$$Register);
+__ add_w($dst$$Register, $y$$Register, $x$$Register);
+__ csel_w($dst$$Register, $dst$$Register, $x$$Register, lt);
+%}
+ins_pipe( cadd_cmpltmask );
+%}
+#else
+instruct cadd_cmpLTMask3( iRegI p, iRegI q, iRegI y, iRegI z, flagsReg ccr ) %{
+match(Set z (AddI (AndI (CmpLTMask p q) y) z));
+effect( KILL ccr );
+ins_cost(DEFAULT_COST*2);
+format %{ "CMP    $p,$q\n\t"
+"ADD.lt $z,$y,$z" %}
+ins_encode %{
+__ cmp($p$$Register, $q$$Register);
+__ add($z$$Register, $y$$Register, $z$$Register, lt);
+%}
+ins_pipe( cadd_cmpltmask );
+%}
+#endif
+#ifdef AARCH64
+instruct cadd_cmpLTMask4( iRegI dst, iRegI p, aimmI q, iRegI y, iRegI x, flagsReg ccr ) %{
+match(Set dst (AddI (AndI (CmpLTMask p q) y) x));
+effect( TEMP dst, KILL ccr );
+size(12);
+ins_cost(DEFAULT_COST*3);
+format %{ "CMP_w  $p,$q\n\t"
+"ADD_w  $dst,$y,$x\n\t"
+"CSEL_w $dst,$dst,$x,lt" %}
+ins_encode %{
+__ cmp_w($p$$Register, $q$$constant);
+__ add_w($dst$$Register, $y$$Register, $x$$Register);
+__ csel_w($dst$$Register, $dst$$Register, $x$$Register, lt);
+%}
+ins_pipe( cadd_cmpltmask );
+%}
+#else
+// FIXME: remove unused "dst"
+instruct cadd_cmpLTMask4( iRegI dst, iRegI p, aimmI q, iRegI y, iRegI z, flagsReg ccr ) %{
+match(Set z (AddI (AndI (CmpLTMask p q) y) z));
+effect( KILL ccr );
+ins_cost(DEFAULT_COST*2);
+format %{ "CMP    $p,$q\n\t"
+"ADD.lt $z,$y,$z" %}
+ins_encode %{
+__ cmp($p$$Register, $q$$constant);
+__ add($z$$Register, $y$$Register, $z$$Register, lt);
+%}
+ins_pipe( cadd_cmpltmask );
+%}
+#endif // !AARCH64
+#ifdef AARCH64
+instruct cadd_cmpLTMask( iRegI dst, iRegI p, iRegI q, iRegI y, flagsReg ccr ) %{
+match(Set dst (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
+effect( TEMP dst, KILL ccr );
+size(12);
+ins_cost(DEFAULT_COST*3);
+format %{ "SUBS_w $p,$p,$q\n\t"
+"ADD_w  $dst,$y,$p\n\t"
+"CSEL_w $dst,$dst,$p,lt" %}
+ins_encode %{
+__ subs_w($p$$Register, $p$$Register, $q$$Register);
+__ add_w($dst$$Register, $y$$Register, $p$$Register);
+__ csel_w($dst$$Register, $dst$$Register, $p$$Register, lt);
+%}
+ins_pipe( cadd_cmpltmask ); // FIXME
+%}
+#else
+instruct cadd_cmpLTMask( iRegI p, iRegI q, iRegI y, flagsReg ccr ) %{
+match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
+effect( KILL ccr );
+ins_cost(DEFAULT_COST*2);
+format %{ "SUBS   $p,$p,$q\n\t"
+"ADD.lt $p,$y,$p" %}
+ins_encode %{
+__ subs($p$$Register, $p$$Register, $q$$Register);
+__ add($p$$Register, $y$$Register, $p$$Register, lt);
+%}
+ins_pipe( cadd_cmpltmask );
+%}
+#endif
+//----------Arithmetic Conversion Instructions---------------------------------
+// The conversions operations are all Alpha sorted.  Please keep it that way!
+instruct convD2F_reg(regF dst, regD src) %{
+match(Set dst (ConvD2F src));
+size(4);
+format %{ "FCVTSD  $dst,$src" %}
+ins_encode %{
+__ convert_d2f($dst$$FloatRegister, $src$$FloatRegister);
+%}
+ins_pipe(fcvtD2F);
+%}
+// Convert a double to an int in a float register.
+// If the double is a NAN, stuff a zero in instead.
+#ifdef AARCH64
+instruct convD2I_reg_reg(iRegI dst, regD src) %{
+match(Set dst (ConvD2I src));
+ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
+format %{ "FCVTZS_wd $dst, $src" %}
+ins_encode %{
+__ fcvtzs_wd($dst$$Register, $src$$FloatRegister);
+%}
+ins_pipe(fcvtD2I);
+%}
+instruct convD2L_reg_reg(iRegL dst, regD src) %{
+match(Set dst (ConvD2L src));
+ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
+format %{ "FCVTZS_xd $dst, $src" %}
+ins_encode %{
+__ fcvtzs_xd($dst$$Register, $src$$FloatRegister);
+%}
+ins_pipe(fcvtD2L);
+%}
+#else
+instruct convD2I_reg_reg(iRegI dst, regD src, regF tmp) %{
+match(Set dst (ConvD2I src));
+effect( TEMP tmp );
+ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
+format %{ "FTOSIZD  $tmp,$src\n\t"
+"FMRS     $dst, $tmp" %}
+ins_encode %{
+__ ftosizd($tmp$$FloatRegister, $src$$FloatRegister);
+__ fmrs($dst$$Register, $tmp$$FloatRegister);
+%}
+ins_pipe(fcvtD2I);
+%}
+#endif
+// Convert a double to a long in a double register.
+// If the double is a NAN, stuff a zero in instead.
+#ifndef AARCH64
+// Double to Long conversion
+instruct convD2L_reg(R0R1RegL dst, regD src) %{
+match(Set dst (ConvD2L src));
+effect(CALL);
+ins_cost(MEMORY_REF_COST); // FIXME
+format %{ "convD2L    $dst,$src\t ! call to SharedRuntime::d2l" %}
+ins_encode %{
+#ifndef __ABI_HARD__
+__ fmrrd($dst$$Register, $dst$$Register->successor(), $src$$FloatRegister);
+#else
+if ($src$$FloatRegister != D0) {
+__ mov_double(D0, $src$$FloatRegister);
+}
+#endif
+address target = CAST_FROM_FN_PTR(address, SharedRuntime::d2l);
+__ call(target, relocInfo::runtime_call_type);
+%}
+ins_pipe(fcvtD2L);
+%}
+#endif
+instruct convF2D_reg(regD dst, regF src) %{
+match(Set dst (ConvF2D src));
+size(4);
+format %{ "FCVTDS  $dst,$src" %}
+ins_encode %{
+__ convert_f2d($dst$$FloatRegister, $src$$FloatRegister);
+%}
+ins_pipe(fcvtF2D);
+%}
+#ifdef AARCH64
+instruct convF2I_reg_reg(iRegI dst, regF src) %{
+match(Set dst (ConvF2I src));
+ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
+size(4);
+format %{ "FCVTZS_ws $dst, $src" %}
+ins_encode %{
+__ fcvtzs_ws($dst$$Register, $src$$FloatRegister);
+%}
+ins_pipe(fcvtF2I);
+%}
+instruct convF2L_reg_reg(iRegL dst, regF src) %{
+match(Set dst (ConvF2L src));
+ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
+size(4);
+format %{ "FCVTZS_xs $dst, $src" %}
+ins_encode %{
+__ fcvtzs_xs($dst$$Register, $src$$FloatRegister);
+%}
+ins_pipe(fcvtF2L);
+%}
+#else
+instruct convF2I_reg_reg(iRegI dst, regF src, regF tmp) %{
+match(Set dst (ConvF2I src));
+effect( TEMP tmp );
+ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
+size(8);
+format %{ "FTOSIZS  $tmp,$src\n\t"
+"FMRS     $dst, $tmp" %}
+ins_encode %{
+__ ftosizs($tmp$$FloatRegister, $src$$FloatRegister);
+__ fmrs($dst$$Register, $tmp$$FloatRegister);
+%}
+ins_pipe(fcvtF2I);
+%}
+// Float to Long conversion
+instruct convF2L_reg(R0R1RegL dst, regF src, R0RegI arg1) %{
+match(Set dst (ConvF2L src));
+ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
+effect(CALL);
+format %{ "convF2L  $dst,$src\t! call to SharedRuntime::f2l" %}
+ins_encode %{
+#ifndef __ABI_HARD__
+__ fmrs($arg1$$Register, $src$$FloatRegister);
+#else
+if($src$$FloatRegister != S0) {
+__ mov_float(S0, $src$$FloatRegister);
+}
+#endif
+address target = CAST_FROM_FN_PTR(address, SharedRuntime::f2l);
+__ call(target, relocInfo::runtime_call_type);
+%}
+ins_pipe(fcvtF2L);
+%}
+#endif
+#ifdef AARCH64
+instruct convI2D_reg_reg(iRegI src, regD dst) %{
+match(Set dst (ConvI2D src));
+ins_cost(DEFAULT_COST + MEMORY_REF_COST); // FIXME
+size(4);
+format %{ "SCVTF_dw $dst,$src" %}
+ins_encode %{
+__ scvtf_dw($dst$$FloatRegister, $src$$Register);
+%}
+ins_pipe(fcvtI2D);
+%}
+#else
+instruct convI2D_reg_reg(iRegI src, regD_low dst) %{
+match(Set dst (ConvI2D src));
+ins_cost(DEFAULT_COST + MEMORY_REF_COST); // FIXME
+size(8);
+format %{ "FMSR     $dst,$src \n\t"
+"FSITOD   $dst $dst"%}
+ins_encode %{
+__ fmsr($dst$$FloatRegister, $src$$Register);
+__ fsitod($dst$$FloatRegister, $dst$$FloatRegister);
+%}
+ins_pipe(fcvtI2D);
+%}
+#endif
+instruct convI2F_reg_reg( regF dst, iRegI src ) %{
+match(Set dst (ConvI2F src));
+ins_cost(DEFAULT_COST + MEMORY_REF_COST); // FIXME
+#ifdef AARCH64
+size(4);
+format %{ "SCVTF_sw $dst,$src" %}
+ins_encode %{
+__ scvtf_sw($dst$$FloatRegister, $src$$Register);
+%}
+#else
+size(8);
+format %{ "FMSR     $dst,$src \n\t"
+"FSITOS   $dst, $dst"%}
+ins_encode %{
+__ fmsr($dst$$FloatRegister, $src$$Register);
+__ fsitos($dst$$FloatRegister, $dst$$FloatRegister);
+%}
+#endif
+ins_pipe(fcvtI2F);
+%}
+instruct convI2L_reg(iRegL dst, iRegI src) %{
+match(Set dst (ConvI2L src));
+#ifdef AARCH64
+size(4);
+format %{ "SXTW   $dst,$src\t! int->long" %}
+ins_encode %{
+__ sxtw($dst$$Register, $src$$Register);
+%}
+#else
+size(8);
+format %{ "MOV    $dst.lo, $src \n\t"
+"ASR    $dst.hi,$src,31\t! int->long" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$Register);
+__ mov($dst$$Register->successor(), AsmOperand($src$$Register, asr, 31));
+%}
+#endif
+ins_pipe(ialu_reg_reg);
+%}
+// Zero-extend convert int to long
+instruct convI2L_reg_zex(iRegL dst, iRegI src, immL_32bits mask ) %{
+match(Set dst (AndL (ConvI2L src) mask) );
+#ifdef AARCH64
+size(4);
+format %{ "mov_w  $dst,$src\t! zero-extend int to long"  %}
+ins_encode %{
+__ mov_w($dst$$Register, $src$$Register);
+%}
+#else
+size(8);
+format %{ "MOV    $dst.lo,$src.lo\t! zero-extend int to long\n\t"
+"MOV    $dst.hi, 0"%}
+ins_encode %{
+__ mov($dst$$Register, $src$$Register);
+__ mov($dst$$Register->successor(), 0);
+%}
+#endif
+ins_pipe(ialu_reg_reg);
+%}
+// Zero-extend long
+instruct zerox_long(iRegL dst, iRegL src, immL_32bits mask ) %{
+match(Set dst (AndL src mask) );
+#ifdef AARCH64
+size(4);
+format %{ "mov_w  $dst,$src\t! zero-extend long"  %}
+ins_encode %{
+__ mov_w($dst$$Register, $src$$Register);
+%}
+#else
+size(8);
+format %{ "MOV    $dst.lo,$src.lo\t! zero-extend long\n\t"
+"MOV    $dst.hi, 0"%}
+ins_encode %{
+__ mov($dst$$Register, $src$$Register);
+__ mov($dst$$Register->successor(), 0);
+%}
+#endif
+ins_pipe(ialu_reg_reg);
+%}
+instruct MoveF2I_reg_reg(iRegI dst, regF src) %{
+match(Set dst (MoveF2I src));
+effect(DEF dst, USE src);
+ins_cost(MEMORY_REF_COST); // FIXME
+size(4);
+format %{ "FMRS   $dst,$src\t! MoveF2I" %}
+ins_encode %{
+__ fmrs($dst$$Register, $src$$FloatRegister);
+%}
+ins_pipe(iload_mem); // FIXME
+%}
+instruct MoveI2F_reg_reg(regF dst, iRegI src) %{
+match(Set dst (MoveI2F src));
+ins_cost(MEMORY_REF_COST); // FIXME
+size(4);
+format %{ "FMSR   $dst,$src\t! MoveI2F" %}
+ins_encode %{
+__ fmsr($dst$$FloatRegister, $src$$Register);
+%}
+ins_pipe(iload_mem); // FIXME
+%}
+instruct MoveD2L_reg_reg(iRegL dst, regD src) %{
+match(Set dst (MoveD2L src));
+effect(DEF dst, USE src);
+ins_cost(MEMORY_REF_COST); // FIXME
+size(4);
+#ifdef AARCH64
+format %{ "FMOV_xd  $dst,$src\t! MoveD2L" %}
+ins_encode %{
+__ fmov_xd($dst$$Register, $src$$FloatRegister);
+%}
+#else
+format %{ "FMRRD    $dst,$src\t! MoveD2L" %}
+ins_encode %{
+__ fmrrd($dst$$Register, $dst$$Register->successor(), $src$$FloatRegister);
+%}
+#endif
+ins_pipe(iload_mem); // FIXME
+%}
+instruct MoveL2D_reg_reg(regD dst, iRegL src) %{
+match(Set dst (MoveL2D src));
+effect(DEF dst, USE src);
+ins_cost(MEMORY_REF_COST); // FIXME
+size(4);
+#ifdef AARCH64
+format %{ "FMOV_dx $dst,$src\t! MoveL2D" %}
+ins_encode %{
+__ fmov_dx($dst$$FloatRegister, $src$$Register);
+%}
+#else
+format %{ "FMDRR   $dst,$src\t! MoveL2D" %}
+ins_encode %{
+__ fmdrr($dst$$FloatRegister, $src$$Register, $src$$Register->successor());
+%}
+#endif
+ins_pipe(ialu_reg_reg); // FIXME
+%}
+//-----------
+// Long to Double conversion
+#ifdef AARCH64
+instruct convL2D(regD dst, iRegL src) %{
+match(Set dst (ConvL2D src));
+ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
+size(4);
+format %{ "SCVTF_dx $dst, $src" %}
+ins_encode %{
+__ scvtf_dx($dst$$FloatRegister, $src$$Register);
+%}
+ins_pipe(fcvtL2D);
+%}
+instruct convL2F(regF dst, iRegL src) %{
+match(Set dst (ConvL2F src));
+ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
+size(4);
+format %{ "SCVTF_sx $dst, $src" %}
+ins_encode %{
+__ scvtf_sx($dst$$FloatRegister, $src$$Register);
+%}
+ins_pipe(fcvtL2F);
+%}
+#else
+// Magic constant, 0x43300000
+instruct loadConI_x43300000(iRegI dst) %{
+effect(DEF dst);
+size(8);
+format %{ "MOV_SLOW  $dst,0x43300000\t! 2^52" %}
+ins_encode %{
+__ mov_slow($dst$$Register, 0x43300000);
+%}
+ins_pipe(ialu_none);
+%}
+// Magic constant, 0x41f00000
+instruct loadConI_x41f00000(iRegI dst) %{
+effect(DEF dst);
+size(8);
+format %{ "MOV_SLOW  $dst, 0x41f00000\t! 2^32" %}
+ins_encode %{
+__ mov_slow($dst$$Register, 0x41f00000);
+%}
+ins_pipe(ialu_none);
+%}
+instruct loadConI_x0(iRegI dst) %{
+effect(DEF dst);
+size(4);
+format %{ "MOV  $dst, 0x0\t! 0" %}
+ins_encode %{
+__ mov($dst$$Register, 0);
+%}
+ins_pipe(ialu_none);
+%}
+// Construct a double from two float halves
+instruct regDHi_regDLo_to_regD(regD_low dst, regD_low src1, regD_low src2) %{
+effect(DEF dst, USE src1, USE src2);
+size(8);
+format %{ "FCPYS  $dst.hi,$src1.hi\n\t"
+"FCPYS  $dst.lo,$src2.lo" %}
+ins_encode %{
+__ fcpys($dst$$FloatRegister->successor(), $src1$$FloatRegister->successor());
+__ fcpys($dst$$FloatRegister, $src2$$FloatRegister);
+%}
+ins_pipe(faddD_reg_reg);
+%}
+#ifndef AARCH64
+// Convert integer in high half of a double register (in the lower half of
+// the double register file) to double
+instruct convI2D_regDHi_regD(regD dst, regD_low src) %{
+effect(DEF dst, USE src);
+size(4);
+format %{ "FSITOD  $dst,$src" %}
+ins_encode %{
+__ fsitod($dst$$FloatRegister, $src$$FloatRegister->successor());
+%}
+ins_pipe(fcvtLHi2D);
+%}
+#endif
+// Add float double precision
+instruct addD_regD_regD(regD dst, regD src1, regD src2) %{
+effect(DEF dst, USE src1, USE src2);
+size(4);
+format %{ "FADDD  $dst,$src1,$src2" %}
+ins_encode %{
+__ add_double($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
+%}
+ins_pipe(faddD_reg_reg);
+%}
+// Sub float double precision
+instruct subD_regD_regD(regD dst, regD src1, regD src2) %{
+effect(DEF dst, USE src1, USE src2);
+size(4);
+format %{ "FSUBD  $dst,$src1,$src2" %}
+ins_encode %{
+__ sub_double($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
+%}
+ins_pipe(faddD_reg_reg);
+%}
+// Mul float double precision
+instruct mulD_regD_regD(regD dst, regD src1, regD src2) %{
+effect(DEF dst, USE src1, USE src2);
+size(4);
+format %{ "FMULD  $dst,$src1,$src2" %}
+ins_encode %{
+__ mul_double($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
+%}
+ins_pipe(fmulD_reg_reg);
+%}
+instruct regL_to_regD(regD dst, iRegL src) %{
+// No match rule to avoid chain rule match.
+effect(DEF dst, USE src);
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "FMDRR   $dst,$src\t! regL to regD" %}
+ins_encode %{
+__ fmdrr($dst$$FloatRegister, $src$$Register, $src$$Register->successor());
+%}
+ins_pipe(ialu_reg_reg); // FIXME
+%}
+instruct regI_regI_to_regD(regD dst, iRegI src1, iRegI src2) %{
+// No match rule to avoid chain rule match.
+effect(DEF dst, USE src1, USE src2);
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "FMDRR   $dst,$src1,$src2\t! regI,regI to regD" %}
+ins_encode %{
+__ fmdrr($dst$$FloatRegister, $src1$$Register, $src2$$Register);
+%}
+ins_pipe(ialu_reg_reg); // FIXME
+%}
+instruct convL2D_reg_slow_fxtof(regD dst, iRegL src) %{
+match(Set dst (ConvL2D src));
+ins_cost(DEFAULT_COST*8 + MEMORY_REF_COST*6); // FIXME
+expand %{
+regD_low   tmpsrc;
+iRegI      ix43300000;
+iRegI      ix41f00000;
+iRegI      ix0;
+regD_low   dx43300000;
+regD       dx41f00000;
+regD       tmp1;
+regD_low   tmp2;
+regD       tmp3;
+regD       tmp4;
+regL_to_regD(tmpsrc, src);
+loadConI_x43300000(ix43300000);
+loadConI_x41f00000(ix41f00000);
+loadConI_x0(ix0);
+regI_regI_to_regD(dx43300000, ix0, ix43300000);
+regI_regI_to_regD(dx41f00000, ix0, ix41f00000);
+convI2D_regDHi_regD(tmp1, tmpsrc);
+regDHi_regDLo_to_regD(tmp2, dx43300000, tmpsrc);
+subD_regD_regD(tmp3, tmp2, dx43300000);
+mulD_regD_regD(tmp4, tmp1, dx41f00000);
+addD_regD_regD(dst, tmp3, tmp4);
+%}
+%}
+#endif // !AARCH64
+instruct convL2I_reg(iRegI dst, iRegL src) %{
+match(Set dst (ConvL2I src));
+size(4);
+#ifdef AARCH64
+format %{ "MOV_w  $dst,$src\t! long->int" %}
+ins_encode %{
+__ mov_w($dst$$Register, $src$$Register);
+%}
+#else
+format %{ "MOV    $dst,$src.lo\t! long->int" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$Register);
+%}
+#endif
+ins_pipe(ialu_move_reg_I_to_L);
+%}
+#ifndef AARCH64
+// Register Shift Right Immediate
+instruct shrL_reg_imm6_L2I(iRegI dst, iRegL src, immI_32_63 cnt) %{
+match(Set dst (ConvL2I (RShiftL src cnt)));
+size(4);
+format %{ "ASR    $dst,$src.hi,($cnt - 32)\t! long->int or mov if $cnt==32" %}
+ins_encode %{
+if ($cnt$$constant == 32) {
+__ mov($dst$$Register, $src$$Register->successor());
+} else {
+__ mov($dst$$Register, AsmOperand($src$$Register->successor(), asr, $cnt$$constant - 32));
+}
+%}
+ins_pipe(ialu_reg_imm);
+%}
+#endif
+//----------Control Flow Instructions------------------------------------------
+// Compare Instructions
+// Compare Integers
+instruct compI_iReg(flagsReg icc, iRegI op1, iRegI op2) %{
+match(Set icc (CmpI op1 op2));
+effect( DEF icc, USE op1, USE op2 );
+size(4);
+format %{ "cmp_32 $op1,$op2\t! int" %}
+ins_encode %{
+__ cmp_32($op1$$Register, $op2$$Register);
+%}
+ins_pipe(ialu_cconly_reg_reg);
+%}
+#ifdef _LP64
+// Compare compressed pointers
+instruct compN_reg2(flagsRegU icc, iRegN op1, iRegN op2) %{
+match(Set icc (CmpN op1 op2));
+effect( DEF icc, USE op1, USE op2 );
+size(4);
+format %{ "cmp_32 $op1,$op2\t! int" %}
+ins_encode %{
+__ cmp_32($op1$$Register, $op2$$Register);
+%}
+ins_pipe(ialu_cconly_reg_reg);
+%}
+#endif
+instruct compU_iReg(flagsRegU icc, iRegI op1, iRegI op2) %{
+match(Set icc (CmpU op1 op2));
+size(4);
+format %{ "cmp_32 $op1,$op2\t! unsigned int" %}
+ins_encode %{
+__ cmp_32($op1$$Register, $op2$$Register);
+%}
+ins_pipe(ialu_cconly_reg_reg);
+%}
+instruct compI_iReg_immneg(flagsReg icc, iRegI op1, aimmIneg op2) %{
+match(Set icc (CmpI op1 op2));
+effect( DEF icc, USE op1 );
+size(4);
+format %{ "cmn_32 $op1,-$op2\t! int" %}
+ins_encode %{
+__ cmn_32($op1$$Register, -$op2$$constant);
+%}
+ins_pipe(ialu_cconly_reg_imm);
+%}
+instruct compI_iReg_imm(flagsReg icc, iRegI op1, aimmI op2) %{
+match(Set icc (CmpI op1 op2));
+effect( DEF icc, USE op1 );
+size(4);
+format %{ "cmp_32 $op1,$op2\t! int" %}
+ins_encode %{
+__ cmp_32($op1$$Register, $op2$$constant);
+%}
+ins_pipe(ialu_cconly_reg_imm);
+%}
+instruct testI_reg_reg( flagsReg_EQNELTGE icc, iRegI op1, iRegI op2, immI0 zero ) %{
+match(Set icc (CmpI (AndI op1 op2) zero));
+size(4);
+format %{ "tst_32 $op2,$op1" %}
+ins_encode %{
+__ tst_32($op1$$Register, $op2$$Register);
+%}
+ins_pipe(ialu_cconly_reg_reg_zero);
+%}
+#ifndef AARCH64
+instruct testshlI_reg_reg_reg( flagsReg_EQNELTGE icc, iRegI op1, iRegI op2, iRegI op3, immI0 zero ) %{
+match(Set icc (CmpI (AndI op1 (LShiftI op2 op3)) zero));
+size(4);
+format %{ "TST   $op2,$op1<<$op3" %}
+ins_encode %{
+__ tst($op1$$Register, AsmOperand($op2$$Register, lsl, $op3$$Register));
+%}
+ins_pipe(ialu_cconly_reg_reg_zero);
+%}
+#endif
+instruct testshlI_reg_reg_imm( flagsReg_EQNELTGE icc, iRegI op1, iRegI op2, immU5 op3, immI0 zero ) %{
+match(Set icc (CmpI (AndI op1 (LShiftI op2 op3)) zero));
+size(4);
+format %{ "tst_32 $op2,$op1<<$op3" %}
+ins_encode %{
+__ tst_32($op1$$Register, AsmOperand($op2$$Register, lsl, $op3$$constant));
+%}
+ins_pipe(ialu_cconly_reg_reg_zero);
+%}
+#ifndef AARCH64
+instruct testsarI_reg_reg_reg( flagsReg_EQNELTGE icc, iRegI op1, iRegI op2, iRegI op3, immI0 zero ) %{
+match(Set icc (CmpI (AndI op1 (RShiftI op2 op3)) zero));
+size(4);
+format %{ "TST   $op2,$op1<<$op3" %}
+ins_encode %{
+__ tst($op1$$Register, AsmOperand($op2$$Register, asr, $op3$$Register));
+%}
+ins_pipe(ialu_cconly_reg_reg_zero);
+%}
+#endif
+instruct testsarI_reg_reg_imm( flagsReg_EQNELTGE icc, iRegI op1, iRegI op2, immU5 op3, immI0 zero ) %{
+match(Set icc (CmpI (AndI op1 (RShiftI op2 op3)) zero));
+size(4);
+format %{ "tst_32 $op2,$op1<<$op3" %}
+ins_encode %{
+__ tst_32($op1$$Register, AsmOperand($op2$$Register, asr, $op3$$constant));
+%}
+ins_pipe(ialu_cconly_reg_reg_zero);
+%}
+#ifndef AARCH64
+instruct testshrI_reg_reg_reg( flagsReg_EQNELTGE icc, iRegI op1, iRegI op2, iRegI op3, immI0 zero ) %{
+match(Set icc (CmpI (AndI op1 (URShiftI op2 op3)) zero));
+size(4);
+format %{ "TST   $op2,$op1<<$op3" %}
+ins_encode %{
+__ tst($op1$$Register, AsmOperand($op2$$Register, lsr, $op3$$Register));
+%}
+ins_pipe(ialu_cconly_reg_reg_zero);
+%}
+#endif
+instruct testshrI_reg_reg_imm( flagsReg_EQNELTGE icc, iRegI op1, iRegI op2, immU5 op3, immI0 zero ) %{
+match(Set icc (CmpI (AndI op1 (URShiftI op2 op3)) zero));
+size(4);
+format %{ "tst_32 $op2,$op1<<$op3" %}
+ins_encode %{
+__ tst_32($op1$$Register, AsmOperand($op2$$Register, lsr, $op3$$constant));
+%}
+ins_pipe(ialu_cconly_reg_reg_zero);
+%}
+instruct testI_reg_imm( flagsReg_EQNELTGE icc, iRegI op1, limmI op2, immI0 zero ) %{
+match(Set icc (CmpI (AndI op1 op2) zero));
+size(4);
+format %{ "tst_32 $op2,$op1" %}
+ins_encode %{
+__ tst_32($op1$$Register, $op2$$constant);
+%}
+ins_pipe(ialu_cconly_reg_imm_zero);
+%}
+#ifdef AARCH64
+instruct compL_reg_reg(flagsReg xcc, iRegL op1, iRegL op2)
+%{
+match(Set xcc (CmpL op1 op2));
+effect( DEF xcc, USE op1, USE op2 );
+size(4);
+format %{ "CMP     $op1,$op2\t! long" %}
+ins_encode %{
+__ cmp($op1$$Register, $op2$$Register);
+%}
+ins_pipe(ialu_cconly_reg_reg);
+%}
+#else
+instruct compL_reg_reg_LTGE(flagsRegL_LTGE xcc, iRegL op1, iRegL op2, iRegL tmp) %{
+match(Set xcc (CmpL op1 op2));
+effect( DEF xcc, USE op1, USE op2, TEMP tmp );
+size(8);
+format %{ "SUBS    $tmp,$op1.low,$op2.low\t\t! long\n\t"
+"SBCS    $tmp,$op1.hi,$op2.hi" %}
+ins_encode %{
+__ subs($tmp$$Register, $op1$$Register, $op2$$Register);
+__ sbcs($tmp$$Register->successor(), $op1$$Register->successor(), $op2$$Register->successor());
+%}
+ins_pipe(ialu_cconly_reg_reg);
+%}
+#endif
+#ifdef AARCH64
+instruct compL_reg_con(flagsReg xcc, iRegL op1, aimmL con) %{
+match(Set xcc (CmpL op1 con));
+effect( DEF xcc, USE op1, USE con );
+size(8);
+format %{ "CMP     $op1,$con\t\t! long"  %}
+ins_encode %{
+__ cmp($op1$$Register, $con$$constant);
+%}
+ins_pipe(ialu_cconly_reg_imm);
+%}
+#else
+instruct compL_reg_reg_EQNE(flagsRegL_EQNE xcc, iRegL op1, iRegL op2) %{
+match(Set xcc (CmpL op1 op2));
+effect( DEF xcc, USE op1, USE op2 );
+size(8);
+format %{ "TEQ    $op1.hi,$op2.hi\t\t! long\n\t"
+"TEQ.eq $op1.lo,$op2.lo" %}
+ins_encode %{
+__ teq($op1$$Register->successor(), $op2$$Register->successor());
+__ teq($op1$$Register, $op2$$Register, eq);
+%}
+ins_pipe(ialu_cconly_reg_reg);
+%}
+instruct compL_reg_reg_LEGT(flagsRegL_LEGT xcc, iRegL op1, iRegL op2, iRegL tmp) %{
+match(Set xcc (CmpL op1 op2));
+effect( DEF xcc, USE op1, USE op2, TEMP tmp );
+size(8);
+format %{ "SUBS    $tmp,$op2.low,$op1.low\t\t! long\n\t"
+"SBCS    $tmp,$op2.hi,$op1.hi" %}
+ins_encode %{
+__ subs($tmp$$Register, $op2$$Register, $op1$$Register);
+__ sbcs($tmp$$Register->successor(), $op2$$Register->successor(), $op1$$Register->successor());
+%}
+ins_pipe(ialu_cconly_reg_reg);
+%}
+// TODO: try immLRot2 instead, (0, $con$$constant) becomes
+// (hi($con$$constant), lo($con$$constant)) becomes
+instruct compL_reg_con_LTGE(flagsRegL_LTGE xcc, iRegL op1, immLlowRot con, iRegL tmp) %{
+match(Set xcc (CmpL op1 con));
+effect( DEF xcc, USE op1, USE con, TEMP tmp );
+size(8);
+format %{ "SUBS    $tmp,$op1.low,$con\t\t! long\n\t"
+"SBCS    $tmp,$op1.hi,0" %}
+ins_encode %{
+__ subs($tmp$$Register, $op1$$Register, $con$$constant);
+__ sbcs($tmp$$Register->successor(), $op1$$Register->successor(), 0);
+%}
+ins_pipe(ialu_cconly_reg_reg);
+%}
+// TODO: try immLRot2 instead, (0, $con$$constant) becomes
+// (hi($con$$constant), lo($con$$constant)) becomes
+instruct compL_reg_con_EQNE(flagsRegL_EQNE xcc, iRegL op1, immLlowRot con) %{
+match(Set xcc (CmpL op1 con));
+effect( DEF xcc, USE op1, USE con );
+size(8);
+format %{ "TEQ    $op1.hi,0\t\t! long\n\t"
+"TEQ.eq $op1.lo,$con" %}
+ins_encode %{
+__ teq($op1$$Register->successor(), 0);
+__ teq($op1$$Register, $con$$constant, eq);
+%}
+ins_pipe(ialu_cconly_reg_reg);
+%}
+// TODO: try immLRot2 instead, (0, $con$$constant) becomes
+// (hi($con$$constant), lo($con$$constant)) becomes
+instruct compL_reg_con_LEGT(flagsRegL_LEGT xcc, iRegL op1, immLlowRot con, iRegL tmp) %{
+match(Set xcc (CmpL op1 con));
+effect( DEF xcc, USE op1, USE con, TEMP tmp );
+size(8);
+format %{ "RSBS    $tmp,$op1.low,$con\t\t! long\n\t"
+"RSCS    $tmp,$op1.hi,0" %}
+ins_encode %{
+__ rsbs($tmp$$Register, $op1$$Register, $con$$constant);
+__ rscs($tmp$$Register->successor(), $op1$$Register->successor(), 0);
+%}
+ins_pipe(ialu_cconly_reg_reg);
+%}
+#endif
+/* instruct testL_reg_reg(flagsRegL xcc, iRegL op1, iRegL op2, immL0 zero) %{ */
+/*   match(Set xcc (CmpL (AndL op1 op2) zero)); */
+/*   ins_encode %{ */
+/*     __ stop("testL_reg_reg unimplemented"); */
+/*   %} */
+/*   ins_pipe(ialu_cconly_reg_reg); */
+/* %} */
+/* // useful for checking the alignment of a pointer: */
+/* instruct testL_reg_con(flagsRegL xcc, iRegL op1, immLlowRot con, immL0 zero) %{ */
+/*   match(Set xcc (CmpL (AndL op1 con) zero)); */
+/*   ins_encode %{ */
+/*     __ stop("testL_reg_con unimplemented"); */
+/*   %} */
+/*   ins_pipe(ialu_cconly_reg_reg); */
+/* %} */
+instruct compU_iReg_imm(flagsRegU icc, iRegI op1, aimmU31 op2 ) %{
+match(Set icc (CmpU op1 op2));
+size(4);
+format %{ "cmp_32 $op1,$op2\t! unsigned" %}
+ins_encode %{
+__ cmp_32($op1$$Register, $op2$$constant);
+%}
+ins_pipe(ialu_cconly_reg_imm);
+%}
+// Compare Pointers
+instruct compP_iRegP(flagsRegP pcc, iRegP op1, iRegP op2 ) %{
+match(Set pcc (CmpP op1 op2));
+size(4);
+format %{ "CMP    $op1,$op2\t! ptr" %}
+ins_encode %{
+__ cmp($op1$$Register, $op2$$Register);
+%}
+ins_pipe(ialu_cconly_reg_reg);
+%}
+instruct compP_iRegP_imm(flagsRegP pcc, iRegP op1, aimmP op2 ) %{
+match(Set pcc (CmpP op1 op2));
+size(4);
+format %{ "CMP    $op1,$op2\t! ptr" %}
+ins_encode %{
+assert($op2$$constant == 0 || _opnds[2]->constant_reloc() == relocInfo::none, "reloc in cmp?");
+__ cmp($op1$$Register, $op2$$constant);
+%}
+ins_pipe(ialu_cconly_reg_imm);
+%}
+//----------Max and Min--------------------------------------------------------
+// Min Instructions
+// Conditional move for min
+instruct cmovI_reg_lt( iRegI op2, iRegI op1, flagsReg icc ) %{
+effect( USE_DEF op2, USE op1, USE icc );
+size(4);
+format %{ "MOV.lt  $op2,$op1\t! min" %}
+ins_encode %{
+__ mov($op2$$Register, $op1$$Register, lt);
+%}
+ins_pipe(ialu_reg_flags);
+%}
+// Min Register with Register.
+instruct minI_eReg(iRegI op1, iRegI op2) %{
+match(Set op2 (MinI op1 op2));
+ins_cost(DEFAULT_COST*2);
+expand %{
+flagsReg icc;
+compI_iReg(icc,op1,op2);
+cmovI_reg_lt(op2,op1,icc);
+%}
+%}
+// Max Instructions
+// Conditional move for max
+instruct cmovI_reg_gt( iRegI op2, iRegI op1, flagsReg icc ) %{
+effect( USE_DEF op2, USE op1, USE icc );
+format %{ "MOV.gt  $op2,$op1\t! max" %}
+ins_encode %{
+__ mov($op2$$Register, $op1$$Register, gt);
+%}
+ins_pipe(ialu_reg_flags);
+%}
+// Max Register with Register
+instruct maxI_eReg(iRegI op1, iRegI op2) %{
+match(Set op2 (MaxI op1 op2));
+ins_cost(DEFAULT_COST*2);
+expand %{
+flagsReg icc;
+compI_iReg(icc,op1,op2);
+cmovI_reg_gt(op2,op1,icc);
+%}
+%}
+//----------Float Compares----------------------------------------------------
+// Compare floating, generate condition code
+instruct cmpF_cc(flagsRegF fcc, flagsReg icc, regF src1, regF src2) %{
+match(Set icc (CmpF src1 src2));
+effect(KILL fcc);
+#ifdef AARCH64
+size(4);
+format %{ "FCMP_s  $src1,$src2" %}
+ins_encode %{
+__ fcmp_s($src1$$FloatRegister, $src2$$FloatRegister);
+%}
+#else
+size(8);
+format %{ "FCMPs  $src1,$src2\n\t"
+"FMSTAT" %}
+ins_encode %{
+__ fcmps($src1$$FloatRegister, $src2$$FloatRegister);
+__ fmstat();
+%}
+#endif
+ins_pipe(faddF_fcc_reg_reg_zero);
+%}
+instruct cmpF0_cc(flagsRegF fcc, flagsReg icc, regF src1, immF0 src2) %{
+match(Set icc (CmpF src1 src2));
+effect(KILL fcc);
+#ifdef AARCH64
+size(4);
+format %{ "FCMP0_s $src1" %}
+ins_encode %{
+__ fcmp0_s($src1$$FloatRegister);
+%}
+#else
+size(8);
+format %{ "FCMPs  $src1,$src2\n\t"
+"FMSTAT" %}
+ins_encode %{
+__ fcmpzs($src1$$FloatRegister);
+__ fmstat();
+%}
+#endif
+ins_pipe(faddF_fcc_reg_reg_zero);
+%}
+instruct cmpD_cc(flagsRegF fcc, flagsReg icc, regD src1, regD src2) %{
+match(Set icc (CmpD src1 src2));
+effect(KILL fcc);
+#ifdef AARCH64
+size(4);
+format %{ "FCMP_d $src1,$src2" %}
+ins_encode %{
+__ fcmp_d($src1$$FloatRegister, $src2$$FloatRegister);
+%}
+#else
+size(8);
+format %{ "FCMPd  $src1,$src2 \n\t"
+"FMSTAT" %}
+ins_encode %{
+__ fcmpd($src1$$FloatRegister, $src2$$FloatRegister);
+__ fmstat();
+%}
+#endif
+ins_pipe(faddD_fcc_reg_reg_zero);
+%}
+instruct cmpD0_cc(flagsRegF fcc, flagsReg icc, regD src1, immD0 src2) %{
+match(Set icc (CmpD src1 src2));
+effect(KILL fcc);
+#ifdef AARCH64
+size(8);
+format %{ "FCMP0_d $src1" %}
+ins_encode %{
+__ fcmp0_d($src1$$FloatRegister);
+%}
+#else
+size(8);
+format %{ "FCMPZd  $src1,$src2 \n\t"
+"FMSTAT" %}
+ins_encode %{
+__ fcmpzd($src1$$FloatRegister);
+__ fmstat();
+%}
+#endif
+ins_pipe(faddD_fcc_reg_reg_zero);
+%}
+#ifdef AARCH64
+// Compare floating, generate -1,0,1
+instruct cmpF_reg(iRegI dst, regF src1, regF src2, flagsReg icc) %{
+match(Set dst (CmpF3 src1 src2));
+// effect(KILL fcc); // nobody cares if flagsRegF is killed
+effect(KILL icc);
+ins_cost(DEFAULT_COST*3); // FIXME
+size(12);
+format %{ "FCMP_s $src1,$src2\n\t"
+"CSET   $dst, gt\n\t"
+"CSINV  $dst, $dst, ZR, ge" %}
+ins_encode %{
+Register dst = $dst$$Register;
+__ fcmp_s($src1$$FloatRegister, $src2$$FloatRegister);
+__ cset(dst, gt);            // 1 if '>', else 0
+__ csinv(dst, dst, ZR, ge);  // previous value if '>=', else -1
+%}
+ins_pipe( floating_cmp ); // FIXME
+%}
+// Compare floating, generate -1,0,1
+instruct cmpD_reg(iRegI dst, regD src1, regD src2, flagsReg icc) %{
+match(Set dst (CmpD3 src1 src2));
+// effect(KILL fcc); // nobody cares if flagsRegF is killed
+effect(KILL icc);
+ins_cost(DEFAULT_COST*3); // FIXME
+size(12);
+format %{ "FCMP_d $src1,$src2\n\t"
+"CSET   $dst, gt\n\t"
+"CSINV  $dst, $dst, ZR, ge" %}
+ins_encode %{
+Register dst = $dst$$Register;
+__ fcmp_d($src1$$FloatRegister, $src2$$FloatRegister);
+__ cset(dst, gt);            // 1 if '>', else 0
+__ csinv(dst, dst, ZR, ge);  // previous value if '>=', else -1
+%}
+ins_pipe( floating_cmp ); // FIXME
+%}
+// Compare floating, generate -1,0,1
+instruct cmpF0_reg(iRegI dst, regF src1, immF0 src2, flagsReg icc) %{
+match(Set dst (CmpF3 src1 src2));
+// effect(KILL fcc); // nobody cares if flagsRegF is killed
+effect(KILL icc);
+ins_cost(DEFAULT_COST*3); // FIXME
+size(12);
+format %{ "FCMP0_s $src1\n\t"
+"CSET   $dst, gt\n\t"
+"CSINV  $dst, $dst, ZR, ge" %}
+ins_encode %{
+Register dst = $dst$$Register;
+__ fcmp0_s($src1$$FloatRegister);
+__ cset(dst, gt);            // 1 if '>', else 0
+__ csinv(dst, dst, ZR, ge);  // previous value if '>=', else -1
+%}
+ins_pipe( floating_cmp ); // FIXME
+%}
+// Compare floating, generate -1,0,1
+instruct cmpD0_reg(iRegI dst, regD src1, immD0 src2, flagsReg icc) %{
+match(Set dst (CmpD3 src1 src2));
+// effect(KILL fcc); // nobody cares if flagsRegF is killed
+effect(KILL icc);
+ins_cost(DEFAULT_COST*3); // FIXME
+size(12);
+format %{ "FCMP0_d $src1\n\t"
+"CSET   $dst, gt\n\t"
+"CSINV  $dst, $dst, ZR, ge" %}
+ins_encode %{
+Register dst = $dst$$Register;
+__ fcmp0_d($src1$$FloatRegister);
+__ cset(dst, gt);            // 1 if '>', else 0
+__ csinv(dst, dst, ZR, ge);  // previous value if '>=', else -1
+%}
+ins_pipe( floating_cmp ); // FIXME
+%}
+#else
+// Compare floating, generate -1,0,1
+instruct cmpF_reg(iRegI dst, regF src1, regF src2, flagsRegF fcc) %{
+match(Set dst (CmpF3 src1 src2));
+effect(KILL fcc);
+ins_cost(DEFAULT_COST*3+BRANCH_COST*3); // FIXME
+size(20);
+// same number of instructions as code using conditional moves but
+// doesn't kill integer condition register
+format %{ "FCMPs  $dst,$src1,$src2 \n\t"
+"VMRS   $dst, FPSCR \n\t"
+"OR     $dst, $dst, 0x08000000 \n\t"
+"EOR    $dst, $dst, $dst << 3 \n\t"
+"MOV    $dst, $dst >> 30" %}
+ins_encode %{
+__ fcmps($src1$$FloatRegister, $src2$$FloatRegister);
+__ floating_cmp($dst$$Register);
+%}
+ins_pipe( floating_cmp );
+%}
+instruct cmpF0_reg(iRegI dst, regF src1, immF0 src2, flagsRegF fcc) %{
+match(Set dst (CmpF3 src1 src2));
+effect(KILL fcc);
+ins_cost(DEFAULT_COST*3+BRANCH_COST*3); // FIXME
+size(20);
+// same number of instructions as code using conditional moves but
+// doesn't kill integer condition register
+format %{ "FCMPZs $dst,$src1,$src2 \n\t"
+"VMRS   $dst, FPSCR \n\t"
+"OR     $dst, $dst, 0x08000000 \n\t"
+"EOR    $dst, $dst, $dst << 3 \n\t"
+"MOV    $dst, $dst >> 30" %}
+ins_encode %{
+__ fcmpzs($src1$$FloatRegister);
+__ floating_cmp($dst$$Register);
+%}
+ins_pipe( floating_cmp );
+%}
+instruct cmpD_reg(iRegI dst, regD src1, regD src2, flagsRegF fcc) %{
+match(Set dst (CmpD3 src1 src2));
+effect(KILL fcc);
+ins_cost(DEFAULT_COST*3+BRANCH_COST*3); // FIXME
+size(20);
+// same number of instructions as code using conditional moves but
+// doesn't kill integer condition register
+format %{ "FCMPd  $dst,$src1,$src2 \n\t"
+"VMRS   $dst, FPSCR \n\t"
+"OR     $dst, $dst, 0x08000000 \n\t"
+"EOR    $dst, $dst, $dst << 3 \n\t"
+"MOV    $dst, $dst >> 30" %}
+ins_encode %{
+__ fcmpd($src1$$FloatRegister, $src2$$FloatRegister);
+__ floating_cmp($dst$$Register);
+%}
+ins_pipe( floating_cmp );
+%}
+instruct cmpD0_reg(iRegI dst, regD src1, immD0 src2, flagsRegF fcc) %{
+match(Set dst (CmpD3 src1 src2));
+effect(KILL fcc);
+ins_cost(DEFAULT_COST*3+BRANCH_COST*3); // FIXME
+size(20);
+// same number of instructions as code using conditional moves but
+// doesn't kill integer condition register
+format %{ "FCMPZd $dst,$src1,$src2 \n\t"
+"VMRS   $dst, FPSCR \n\t"
+"OR     $dst, $dst, 0x08000000 \n\t"
+"EOR    $dst, $dst, $dst << 3 \n\t"
+"MOV    $dst, $dst >> 30" %}
+ins_encode %{
+__ fcmpzd($src1$$FloatRegister);
+__ floating_cmp($dst$$Register);
+%}
+ins_pipe( floating_cmp );
+%}
+#endif // !AARCH64
+//----------Branches---------------------------------------------------------
+// Jump
+// (compare 'operand indIndex' and 'instruct addP_reg_reg' above)
+// FIXME
+instruct jumpXtnd(iRegX switch_val, iRegP tmp) %{
+match(Jump switch_val);
+effect(TEMP tmp);
+ins_cost(350);
+format %{  "ADD    $tmp, $constanttablebase, $switch_val\n\t"
+"LDR    $tmp,[$tmp + $constantoffset]\n\t"
+"BX     $tmp" %}
+size(20);
+ins_encode %{
+Register table_reg;
+Register label_reg = $tmp$$Register;
+if (constant_offset() == 0) {
+table_reg = $constanttablebase;
+__ ldr(label_reg, Address(table_reg, $switch_val$$Register));
+} else {
+table_reg = $tmp$$Register;
+int offset = $constantoffset;
+if (is_memoryP(offset)) {
+__ add(table_reg, $constanttablebase, $switch_val$$Register);
+__ ldr(label_reg, Address(table_reg, offset));
+} else {
+__ mov_slow(table_reg, $constantoffset);
+__ add(table_reg, $constanttablebase, table_reg);
+__ ldr(label_reg, Address(table_reg, $switch_val$$Register));
+}
+}
+__ jump(label_reg); // ldr + b better than ldr to PC for branch predictor?
+//    __ ldr(PC, Address($table$$Register, $switch_val$$Register));
+%}
+ins_pipe(ialu_reg_reg);
+%}
+// // Direct Branch.
+instruct branch(label labl) %{
+match(Goto);
+effect(USE labl);
+size(4);
+ins_cost(BRANCH_COST);
+format %{ "B     $labl" %}
+ins_encode %{
+__ b(*($labl$$label));
+%}
+ins_pipe(br);
+%}
+// Conditional Direct Branch
+instruct branchCon(cmpOp cmp, flagsReg icc, label labl) %{
+match(If cmp icc);
+effect(USE labl);
+size(4);
+ins_cost(BRANCH_COST);
+format %{ "B$cmp   $icc,$labl" %}
+ins_encode %{
+__ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(br_cc);
+%}
+#ifdef ARM
+instruct branchCon_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, label labl) %{
+match(If cmp icc);
+effect(USE labl);
+predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
+size(4);
+ins_cost(BRANCH_COST);
+format %{ "B$cmp   $icc,$labl" %}
+ins_encode %{
+__ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(br_cc);
+%}
+#endif
+#ifdef AARCH64
+instruct cbzI(cmpOp cmp, iRegI op1, immI0 op2, label labl) %{
+match(If cmp (CmpI op1 op2));
+effect(USE labl);
+predicate(_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
+_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne);
+size(4);
+ins_cost(BRANCH_COST);
+format %{ "CB{N}Z $op1, $labl\t! int $cmp" %}
+ins_encode %{
+if ($cmp$$cmpcode == eq) {
+__ cbz_w($op1$$Register, *($labl$$label));
+} else {
+__ cbnz_w($op1$$Register, *($labl$$label));
+}
+%}
+ins_pipe(br_cc); // FIXME
+%}
+instruct cbzP(cmpOpP cmp, iRegP op1, immP0 op2, label labl) %{
+match(If cmp (CmpP op1 op2));
+effect(USE labl);
+predicate(_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
+_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne);
+size(4);
+ins_cost(BRANCH_COST);
+format %{ "CB{N}Z $op1, $labl\t! ptr $cmp" %}
+ins_encode %{
+if ($cmp$$cmpcode == eq) {
+__ cbz($op1$$Register, *($labl$$label));
+} else {
+__ cbnz($op1$$Register, *($labl$$label));
+}
+%}
+ins_pipe(br_cc); // FIXME
+%}
+instruct cbzL(cmpOpL cmp, iRegL op1, immL0 op2, label labl) %{
+match(If cmp (CmpL op1 op2));
+effect(USE labl);
+predicate(_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
+_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne);
+size(4);
+ins_cost(BRANCH_COST);
+format %{ "CB{N}Z $op1, $labl\t! long $cmp" %}
+ins_encode %{
+if ($cmp$$cmpcode == eq) {
+__ cbz($op1$$Register, *($labl$$label));
+} else {
+__ cbnz($op1$$Register, *($labl$$label));
+}
+%}
+ins_pipe(br_cc); // FIXME
+%}
+#endif
+instruct branchConU(cmpOpU cmp, flagsRegU icc, label labl) %{
+match(If cmp icc);
+effect(USE labl);
+size(4);
+ins_cost(BRANCH_COST);
+format %{ "B$cmp  $icc,$labl" %}
+ins_encode %{
+__ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(br_cc);
+%}
+instruct branchConP(cmpOpP cmp, flagsRegP pcc, label labl) %{
+match(If cmp pcc);
+effect(USE labl);
+size(4);
+ins_cost(BRANCH_COST);
+format %{ "B$cmp  $pcc,$labl" %}
+ins_encode %{
+__ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(br_cc);
+%}
+#ifndef AARCH64
+instruct branchConL_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, label labl) %{
+match(If cmp xcc);
+effect(USE labl);
+predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
+size(4);
+ins_cost(BRANCH_COST);
+format %{ "B$cmp  $xcc,$labl" %}
+ins_encode %{
+__ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(br_cc);
+%}
+instruct branchConL_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, label labl) %{
+match(If cmp xcc);
+effect(USE labl);
+predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
+size(4);
+ins_cost(BRANCH_COST);
+format %{ "B$cmp  $xcc,$labl" %}
+ins_encode %{
+__ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(br_cc);
+%}
+instruct branchConL_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, label labl) %{
+match(If cmp xcc);
+effect(USE labl);
+predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le );
+size(4);
+ins_cost(BRANCH_COST);
+format %{ "B$cmp  $xcc,$labl" %}
+ins_encode %{
+__ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(br_cc);
+%}
+#endif
+instruct branchLoopEnd(cmpOp cmp, flagsReg icc, label labl) %{
+match(CountedLoopEnd cmp icc);
+effect(USE labl);
+size(4);
+ins_cost(BRANCH_COST);
+format %{ "B$cmp   $icc,$labl\t! Loop end" %}
+ins_encode %{
+__ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(br_cc);
+%}
+// instruct branchLoopEndU(cmpOpU cmp, flagsRegU icc, label labl) %{
+//   match(CountedLoopEnd cmp icc);
+//   ins_pipe(br_cc);
+// %}
+// ============================================================================
+// Long Compare
+//
+// Currently we hold longs in 2 registers.  Comparing such values efficiently
+// is tricky.  The flavor of compare used depends on whether we are testing
+// for LT, LE, or EQ.  For a simple LT test we can check just the sign bit.
+// The GE test is the negated LT test.  The LE test can be had by commuting
+// the operands (yielding a GE test) and then negating; negate again for the
+// GT test.  The EQ test is done by ORcc'ing the high and low halves, and the
+// NE test is negated from that.
+// Due to a shortcoming in the ADLC, it mixes up expressions like:
+// (foo (CmpI (CmpL X Y) 0)) and (bar (CmpI (CmpL X 0L) 0)).  Note the
+// difference between 'Y' and '0L'.  The tree-matches for the CmpI sections
+// are collapsed internally in the ADLC's dfa-gen code.  The match for
+// (CmpI (CmpL X Y) 0) is silently replaced with (CmpI (CmpL X 0L) 0) and the
+// foo match ends up with the wrong leaf.  One fix is to not match both
+// reg-reg and reg-zero forms of long-compare.  This is unfortunate because
+// both forms beat the trinary form of long-compare and both are very useful
+// on Intel which has so few registers.
+// instruct branchCon_long(cmpOp cmp, flagsRegL xcc, label labl) %{
+//   match(If cmp xcc);
+//   ins_pipe(br_cc);
+// %}
+// Manifest a CmpL3 result in an integer register.  Very painful.
+// This is the test to avoid.
+#ifdef AARCH64
+instruct cmpL3_reg_reg(iRegI dst, iRegL src1, iRegL src2, flagsReg ccr) %{
+match(Set dst (CmpL3 src1 src2));
+// effect(KILL fcc); // nobody cares if flagsRegF is killed
+effect(KILL ccr);
+ins_cost(DEFAULT_COST*3); // FIXME
+size(12);
+format %{ "CMP    $src1,$src2\n\t"
+"CSET   $dst, gt\n\t"
+"CSINV  $dst, $dst, ZR, ge" %}
+ins_encode %{
+Register dst = $dst$$Register;
+__ cmp($src1$$Register, $src2$$Register);
+__ cset(dst, gt);            // 1 if '>', else 0
+__ csinv(dst, dst, ZR, ge);  // previous value if '>=', else -1
+%}
+ins_pipe( ialu_cconly_reg_reg ); // FIXME
+%}
+// TODO cmpL3_reg_imm
+#else
+instruct cmpL3_reg_reg(iRegI dst, iRegL src1, iRegL src2, flagsReg ccr ) %{
+match(Set dst (CmpL3 src1 src2) );
+effect( KILL ccr );
+ins_cost(6*DEFAULT_COST); // FIXME
+size(32);
+format %{
+"CMP    $src1.hi, $src2.hi\t\t! long\n"
+"\tMOV.gt $dst, 1\n"
+"\tmvn.lt $dst, 0\n"
+"\tB.ne   done\n"
+"\tSUBS   $dst, $src1.lo, $src2.lo\n"
+"\tMOV.hi $dst, 1\n"
+"\tmvn.lo $dst, 0\n"
+"done:"     %}
+ins_encode %{
+Label done;
+__ cmp($src1$$Register->successor(), $src2$$Register->successor());
+__ mov($dst$$Register, 1, gt);
+__ mvn($dst$$Register, 0, lt);
+__ b(done, ne);
+__ subs($dst$$Register, $src1$$Register, $src2$$Register);
+__ mov($dst$$Register, 1, hi);
+__ mvn($dst$$Register, 0, lo);
+__ bind(done);
+%}
+ins_pipe(cmpL_reg);
+%}
+#endif
+#ifndef AARCH64
+// Conditional move
+instruct cmovLL_reg_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, iRegL dst, iRegL src) %{
+match(Set dst (CMoveL (Binary cmp xcc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
+ins_cost(150);
+size(8);
+format %{ "MOV$cmp  $dst.lo,$src.lo\t! long\n\t"
+"MOV$cmp  $dst,$src.hi" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
+__ mov($dst$$Register->successor(), $src$$Register->successor(), (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+instruct cmovLL_reg_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, iRegL dst, iRegL src) %{
+match(Set dst (CMoveL (Binary cmp xcc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
+ins_cost(150);
+size(8);
+format %{ "MOV$cmp  $dst.lo,$src.lo\t! long\n\t"
+"MOV$cmp  $dst,$src.hi" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
+__ mov($dst$$Register->successor(), $src$$Register->successor(), (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+instruct cmovLL_reg_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, iRegL dst, iRegL src) %{
+match(Set dst (CMoveL (Binary cmp xcc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
+ins_cost(150);
+size(8);
+format %{ "MOV$cmp  $dst.lo,$src.lo\t! long\n\t"
+"MOV$cmp  $dst,$src.hi" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
+__ mov($dst$$Register->successor(), $src$$Register->successor(), (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+instruct cmovLL_imm_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, iRegL dst, immL0 src) %{
+match(Set dst (CMoveL (Binary cmp xcc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
+ins_cost(140);
+size(8);
+format %{ "MOV$cmp  $dst.lo,0\t! long\n\t"
+"MOV$cmp  $dst,0" %}
+ins_encode %{
+__ mov($dst$$Register, 0, (AsmCondition)($cmp$$cmpcode));
+__ mov($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_imm);
+%}
+instruct cmovLL_imm_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, iRegL dst, immL0 src) %{
+match(Set dst (CMoveL (Binary cmp xcc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
+ins_cost(140);
+size(8);
+format %{ "MOV$cmp  $dst.lo,0\t! long\n\t"
+"MOV$cmp  $dst,0" %}
+ins_encode %{
+__ mov($dst$$Register, 0, (AsmCondition)($cmp$$cmpcode));
+__ mov($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_imm);
+%}
+instruct cmovLL_imm_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, iRegL dst, immL0 src) %{
+match(Set dst (CMoveL (Binary cmp xcc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
+ins_cost(140);
+size(8);
+format %{ "MOV$cmp  $dst.lo,0\t! long\n\t"
+"MOV$cmp  $dst,0" %}
+ins_encode %{
+__ mov($dst$$Register, 0, (AsmCondition)($cmp$$cmpcode));
+__ mov($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_imm);
+%}
+#endif // !AARCH64
+#ifndef AARCH64
+instruct cmovIL_reg_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, iRegI dst, iRegI src) %{
+match(Set dst (CMoveI (Binary cmp xcc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
+ins_cost(150);
+size(4);
+format %{ "MOV$cmp  $dst,$src" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+instruct cmovIL_reg_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, iRegI dst, iRegI src) %{
+match(Set dst (CMoveI (Binary cmp xcc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
+ins_cost(150);
+size(4);
+format %{ "MOV$cmp  $dst,$src" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+instruct cmovIL_reg_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, iRegI dst, iRegI src) %{
+match(Set dst (CMoveI (Binary cmp xcc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
+ins_cost(150);
+size(4);
+format %{ "MOV$cmp  $dst,$src" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+#endif // !AARCH64
+#ifndef AARCH64
+instruct cmovIL_imm_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, iRegI dst, immI16 src) %{
+match(Set dst (CMoveI (Binary cmp xcc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
+ins_cost(140);
+format %{ "MOVW$cmp  $dst,$src" %}
+ins_encode %{
+__ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_imm);
+%}
+instruct cmovIL_imm_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, iRegI dst, immI16 src) %{
+match(Set dst (CMoveI (Binary cmp xcc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
+ins_cost(140);
+format %{ "MOVW$cmp  $dst,$src" %}
+ins_encode %{
+__ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_imm);
+%}
+instruct cmovIL_imm_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, iRegI dst, immI16 src) %{
+match(Set dst (CMoveI (Binary cmp xcc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
+ins_cost(140);
+format %{ "MOVW$cmp  $dst,$src" %}
+ins_encode %{
+__ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_imm);
+%}
+instruct cmovPL_reg_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, iRegP dst, iRegP src) %{
+match(Set dst (CMoveP (Binary cmp xcc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
+ins_cost(150);
+size(4);
+format %{ "MOV$cmp  $dst,$src" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+instruct cmovPL_reg_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, iRegP dst, iRegP src) %{
+match(Set dst (CMoveP (Binary cmp xcc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
+ins_cost(150);
+size(4);
+format %{ "MOV$cmp  $dst,$src" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+instruct cmovPL_reg_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, iRegP dst, iRegP src) %{
+match(Set dst (CMoveP (Binary cmp xcc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
+ins_cost(150);
+size(4);
+format %{ "MOV$cmp  $dst,$src" %}
+ins_encode %{
+__ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_reg);
+%}
+instruct cmovPL_imm_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, iRegP dst, immP0 src) %{
+match(Set dst (CMoveP (Binary cmp xcc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
+ins_cost(140);
+format %{ "MOVW$cmp  $dst,$src" %}
+ins_encode %{
+__ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_imm);
+%}
+instruct cmovPL_imm_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, iRegP dst, immP0 src) %{
+match(Set dst (CMoveP (Binary cmp xcc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
+ins_cost(140);
+format %{ "MOVW$cmp  $dst,$src" %}
+ins_encode %{
+__ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_imm);
+%}
+instruct cmovPL_imm_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, iRegP dst, immP0 src) %{
+match(Set dst (CMoveP (Binary cmp xcc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
+ins_cost(140);
+format %{ "MOVW$cmp  $dst,$src" %}
+ins_encode %{
+__ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(ialu_imm);
+%}
+instruct cmovFL_reg_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, regF dst, regF src) %{
+match(Set dst (CMoveF (Binary cmp xcc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
+ins_cost(150);
+size(4);
+format %{ "FCPYS$cmp $dst,$src" %}
+ins_encode %{
+__ fcpys($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(int_conditional_float_move);
+%}
+instruct cmovFL_reg_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, regF dst, regF src) %{
+match(Set dst (CMoveF (Binary cmp xcc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
+ins_cost(150);
+size(4);
+format %{ "FCPYS$cmp $dst,$src" %}
+ins_encode %{
+__ fcpys($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(int_conditional_float_move);
+%}
+instruct cmovFL_reg_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, regF dst, regF src) %{
+match(Set dst (CMoveF (Binary cmp xcc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
+ins_cost(150);
+size(4);
+format %{ "FCPYS$cmp $dst,$src" %}
+ins_encode %{
+__ fcpys($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(int_conditional_float_move);
+%}
+instruct cmovDL_reg_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, regD dst, regD src) %{
+match(Set dst (CMoveD (Binary cmp xcc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
+ins_cost(150);
+size(4);
+format %{ "FCPYD$cmp $dst,$src" %}
+ins_encode %{
+__ fcpyd($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(int_conditional_float_move);
+%}
+instruct cmovDL_reg_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, regD dst, regD src) %{
+match(Set dst (CMoveD (Binary cmp xcc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
+ins_cost(150);
+size(4);
+format %{ "FCPYD$cmp $dst,$src" %}
+ins_encode %{
+__ fcpyd($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(int_conditional_float_move);
+%}
+instruct cmovDL_reg_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, regD dst, regD src) %{
+match(Set dst (CMoveD (Binary cmp xcc) (Binary dst src)));
+predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
+ins_cost(150);
+size(4);
+format %{ "FCPYD$cmp $dst,$src" %}
+ins_encode %{
+__ fcpyd($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
+%}
+ins_pipe(int_conditional_float_move);
+%}
+#endif // !AARCH64
+// ============================================================================
+// Safepoint Instruction
+#ifdef AARCH64
+instruct safePoint_poll(iRegP poll, flagsReg icc, RtempRegP tmp) %{
+match(SafePoint poll);
+// The handler stub kills Rtemp
+effect(USE poll, KILL tmp, KILL icc);
+size(4);
+format %{ "LDR   ZR,[$poll]\t! Safepoint: poll for GC" %}
+ins_encode %{
+__ relocate(relocInfo::poll_type);
+__ ldr(ZR, Address($poll$$Register));
+%}
+ins_pipe(loadPollP);
+%}
+#else
+// rather than KILL R12, it would be better to use any reg as
+// TEMP. Can't do that at this point because it crashes the compiler
+instruct safePoint_poll(iRegP poll, R12RegI tmp, flagsReg icc) %{
+match(SafePoint poll);
+effect(USE poll, KILL tmp, KILL icc);
+size(4);
+format %{ "LDR   $tmp,[$poll]\t! Safepoint: poll for GC" %}
+ins_encode %{
+__ relocate(relocInfo::poll_type);
+__ ldr($tmp$$Register, Address($poll$$Register));
+%}
+ins_pipe(loadPollP);
+%}
+#endif
+// ============================================================================
+// Call Instructions
+// Call Java Static Instruction
+instruct CallStaticJavaDirect( method meth ) %{
+match(CallStaticJava);
+predicate(! ((CallStaticJavaNode*)n)->is_method_handle_invoke());
+effect(USE meth);
+ins_cost(CALL_COST);
+format %{ "CALL,static ==> " %}
+ins_encode( Java_Static_Call( meth ), call_epilog );
+ins_pipe(simple_call);
+%}
+// Call Java Static Instruction (method handle version)
+instruct CallStaticJavaHandle( method meth ) %{
+match(CallStaticJava);
+predicate(((CallStaticJavaNode*)n)->is_method_handle_invoke());
+effect(USE meth);
+// FP is saved by all callees (for interpreter stack correction).
+// We use it here for a similar purpose, in {preserve,restore}_FP.
+ins_cost(CALL_COST);
+format %{ "CALL,static/MethodHandle ==> " %}
+ins_encode( preserve_SP, Java_Static_Call( meth ), restore_SP, call_epilog );
+ins_pipe(simple_call);
+%}
+// Call Java Dynamic Instruction
+instruct CallDynamicJavaDirect( method meth ) %{
+match(CallDynamicJava);
+effect(USE meth);
+ins_cost(CALL_COST);
+format %{ "MOV_OOP    (empty),R_R8\n\t"
+"CALL,dynamic  ; NOP ==> " %}
+ins_encode( Java_Dynamic_Call( meth ), call_epilog );
+ins_pipe(call);
+%}
+// Call Runtime Instruction
+instruct CallRuntimeDirect(method meth) %{
+match(CallRuntime);
+effect(USE meth);
+ins_cost(CALL_COST);
+format %{ "CALL,runtime" %}
+#ifdef AARCH64
+ins_encode( save_last_PC, Java_To_Runtime( meth ),
+call_epilog );
+#else
+ins_encode( Java_To_Runtime( meth ),
+call_epilog );
+#endif
+ins_pipe(simple_call);
+%}
+// Call runtime without safepoint - same as CallRuntime
+instruct CallLeafDirect(method meth) %{
+match(CallLeaf);
+effect(USE meth);
+ins_cost(CALL_COST);
+format %{ "CALL,runtime leaf" %}
+// TODO: ned save_last_PC here?
+ins_encode( Java_To_Runtime( meth ),
+call_epilog );
+ins_pipe(simple_call);
+%}
+// Call runtime without safepoint - same as CallLeaf
+instruct CallLeafNoFPDirect(method meth) %{
+match(CallLeafNoFP);
+effect(USE meth);
+ins_cost(CALL_COST);
+format %{ "CALL,runtime leaf nofp" %}
+// TODO: ned save_last_PC here?
+ins_encode( Java_To_Runtime( meth ),
+call_epilog );
+ins_pipe(simple_call);
+%}
+// Tail Call; Jump from runtime stub to Java code.
+// Also known as an 'interprocedural jump'.
+// Target of jump will eventually return to caller.
+// TailJump below removes the return address.
+instruct TailCalljmpInd(IPRegP jump_target, inline_cache_regP method_oop) %{
+match(TailCall jump_target method_oop );
+ins_cost(CALL_COST);
+format %{ "MOV    Rexception_pc, LR\n\t"
+"jump   $jump_target  \t! $method_oop holds method oop" %}
+ins_encode %{
+__ mov(Rexception_pc, LR);   // this is used only to call
+// StubRoutines::forward_exception_entry()
+// which expects PC of exception in
+// R5. FIXME?
+__ jump($jump_target$$Register);
+%}
+ins_pipe(tail_call);
+%}
+// Return Instruction
+instruct Ret() %{
+match(Return);
+format %{ "ret LR" %}
+ins_encode %{
+__ ret(LR);
+%}
+ins_pipe(br);
+%}
+// Tail Jump; remove the return address; jump to target.
+// TailCall above leaves the return address around.
+// TailJump is used in only one place, the rethrow_Java stub (fancy_jump=2).
+// ex_oop (Exception Oop) is needed in %o0 at the jump. As there would be a
+// "restore" before this instruction (in Epilogue), we need to materialize it
+// in %i0.
+instruct tailjmpInd(IPRegP jump_target, RExceptionRegP ex_oop) %{
+match( TailJump jump_target ex_oop );
+ins_cost(CALL_COST);
+format %{ "MOV    Rexception_pc, LR\n\t"
+"jump   $jump_target \t! $ex_oop holds exc. oop" %}
+ins_encode %{
+__ mov(Rexception_pc, LR);
+__ jump($jump_target$$Register);
+%}
+ins_pipe(tail_call);
+%}
+// Create exception oop: created by stack-crawling runtime code.
+// Created exception is now available to this handler, and is setup
+// just prior to jumping to this handler.  No code emitted.
+instruct CreateException( RExceptionRegP ex_oop )
+%{
+match(Set ex_oop (CreateEx));
+ins_cost(0);
+size(0);
+// use the following format syntax
+format %{ "! exception oop is in Rexception_obj; no code emitted" %}
+ins_encode();
+ins_pipe(empty);
+%}
+// Rethrow exception:
+// The exception oop will come in the first argument position.
+// Then JUMP (not call) to the rethrow stub code.
+instruct RethrowException()
+%{
+match(Rethrow);
+ins_cost(CALL_COST);
+// use the following format syntax
+format %{ "b    rethrow_stub" %}
+ins_encode %{
+Register scratch = R1_tmp;
+assert_different_registers(scratch, c_rarg0, LR);
+__ jump(OptoRuntime::rethrow_stub(), relocInfo::runtime_call_type, scratch);
+%}
+ins_pipe(tail_call);
+%}
+// Die now
+instruct ShouldNotReachHere( )
+%{
+match(Halt);
+ins_cost(CALL_COST);
+size(4);
+// Use the following format syntax
+format %{ "breakpoint   ; ShouldNotReachHere" %}
+ins_encode %{
+__ breakpoint();
+%}
+ins_pipe(tail_call);
+%}
+// ============================================================================
+// The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary superklass
+// array for an instance of the superklass.  Set a hidden internal cache on a
+// hit (cache is checked with exposed code in gen_subtype_check()).  Return
+// not zero for a miss or zero for a hit.  The encoding ALSO sets flags.
+instruct partialSubtypeCheck( R0RegP index, R1RegP sub, R2RegP super, flagsRegP pcc, LRRegP lr ) %{
+match(Set index (PartialSubtypeCheck sub super));
+effect( KILL pcc, KILL lr );
+ins_cost(DEFAULT_COST*10);
+format %{ "CALL   PartialSubtypeCheck" %}
+ins_encode %{
+__ call(StubRoutines::Arm::partial_subtype_check(), relocInfo::runtime_call_type);
+%}
+ins_pipe(partial_subtype_check_pipe);
+%}
+/* instruct partialSubtypeCheck_vs_zero( flagsRegP pcc, o1RegP sub, o2RegP super, immP0 zero, o0RegP idx, o7RegP o7 ) %{ */
+/*   match(Set pcc (CmpP (PartialSubtypeCheck sub super) zero)); */
+/*   ins_pipe(partial_subtype_check_pipe); */
+/* %} */
+// ============================================================================
+// inlined locking and unlocking
+#ifdef AARCH64
+instruct cmpFastLock(flagsRegP pcc, iRegP object, iRegP box, iRegP scratch2, iRegP scratch, iRegP scratch3 )
+#else
+instruct cmpFastLock(flagsRegP pcc, iRegP object, iRegP box, iRegP scratch2, iRegP scratch )
+#endif
+%{
+match(Set pcc (FastLock object box));
+#ifdef AARCH64
+effect(TEMP scratch, TEMP scratch2, TEMP scratch3);
+#else
+effect(TEMP scratch, TEMP scratch2);
+#endif
+ins_cost(100);
+#ifdef AARCH64
+format %{ "FASTLOCK  $object, $box; KILL $scratch, $scratch2, $scratch3" %}
+ins_encode %{
+__ fast_lock($object$$Register, $box$$Register, $scratch$$Register, $scratch2$$Register, $scratch3$$Register);
+%}
+#else
+format %{ "FASTLOCK  $object, $box; KILL $scratch, $scratch2" %}
+ins_encode %{
+__ fast_lock($object$$Register, $box$$Register, $scratch$$Register, $scratch2$$Register);
+%}
+#endif
+ins_pipe(long_memory_op);
+%}
+#ifdef AARCH64
+instruct cmpFastUnlock(flagsRegP pcc, iRegP object, iRegP box, iRegP scratch2, iRegP scratch, iRegP scratch3 ) %{
+match(Set pcc (FastUnlock object box));
+effect(TEMP scratch, TEMP scratch2, TEMP scratch3);
+ins_cost(100);
+format %{ "FASTUNLOCK  $object, $box; KILL $scratch, $scratch2, $scratch3" %}
+ins_encode %{
+__ fast_unlock($object$$Register, $box$$Register, $scratch$$Register, $scratch2$$Register, $scratch3$$Register);
+%}
+ins_pipe(long_memory_op);
+%}
+#else
+instruct cmpFastUnlock(flagsRegP pcc, iRegP object, iRegP box, iRegP scratch2, iRegP scratch ) %{
+match(Set pcc (FastUnlock object box));
+effect(TEMP scratch, TEMP scratch2);
+ins_cost(100);
+format %{ "FASTUNLOCK  $object, $box; KILL $scratch, $scratch2" %}
+ins_encode %{
+__ fast_unlock($object$$Register, $box$$Register, $scratch$$Register, $scratch2$$Register);
+%}
+ins_pipe(long_memory_op);
+%}
+#endif
+#ifdef AARCH64
+// TODO: add version that takes immI cnt?
+instruct clear_array(iRegX cnt, iRegP base, iRegP ptr, iRegX temp, Universe dummy, flagsReg cpsr) %{
+match(Set dummy (ClearArray cnt base));
+effect(TEMP temp, TEMP ptr, KILL cpsr);
+ins_cost(300);
+format %{
+"        MOV    $temp,$cnt\n"
+"        ADD    $ptr,$base,$cnt\n"
+"        SUBS   $temp,$temp,16\t! Count down dword pair in bytes\n"
+"        B.lt   done16\n"
+"loop:   STP    ZR,ZR,[$ptr,-16]!\n"
+"        SUBS   $temp,$temp,16\t! Count down dword pair in bytes\n"
+"        B.ge   loop\t! Clearing loop\n"
+"done16: ADDS   $temp,$temp,8\t! Room for 1 more long?\n"
+"        B.lt   done\n"
+"        STR    ZR,[$base+$temp]\n"
+"done:"
+%}
+ins_encode %{
+// TODO: preload?
+__ mov($temp$$Register, $cnt$$Register);
+__ add($ptr$$Register, $base$$Register, $cnt$$Register);
+Label loop, done, done16;
+__ subs($temp$$Register, $temp$$Register, 16);
+__ b(done16, lt);
+__ bind(loop);
+__ stp(ZR, ZR, Address($ptr$$Register, -16, pre_indexed));
+__ subs($temp$$Register, $temp$$Register, 16);
+__ b(loop, ge);
+__ bind(done16);
+__ adds($temp$$Register, $temp$$Register, 8);
+__ b(done, lt);
+// $temp should be 0 here
+__ str(ZR, Address($base$$Register, $temp$$Register));
+__ bind(done);
+%}
+ins_pipe(long_memory_op);
+%}
+#else
+// Count and Base registers are fixed because the allocator cannot
+// kill unknown registers.  The encodings are generic.
+instruct clear_array(iRegX cnt, iRegP base, iRegI temp, iRegX zero, Universe dummy, flagsReg cpsr) %{
+match(Set dummy (ClearArray cnt base));
+effect(TEMP temp, TEMP zero, KILL cpsr);
+ins_cost(300);
+format %{ "MOV    $zero,0\n"
+"        MOV    $temp,$cnt\n"
+"loop:   SUBS   $temp,$temp,4\t! Count down a dword of bytes\n"
+"        STR.ge $zero,[$base+$temp]\t! delay slot"
+"        B.gt   loop\t\t! Clearing loop\n" %}
+ins_encode %{
+__ mov($zero$$Register, 0);
+__ mov($temp$$Register, $cnt$$Register);
+Label(loop);
+__ bind(loop);
+__ subs($temp$$Register, $temp$$Register, 4);
+__ str($zero$$Register, Address($base$$Register, $temp$$Register), ge);
+__ b(loop, gt);
+%}
+ins_pipe(long_memory_op);
+%}
+#endif
+#ifdef XXX
+// FIXME: Why R0/R1/R2/R3?
+instruct string_compare(R0RegP str1, R1RegP str2, R2RegI cnt1, R3RegI cnt2, iRegI result,
+iRegI tmp1, iRegI tmp2, flagsReg ccr) %{
+predicate(!CompactStrings);
+match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
+effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL ccr, TEMP tmp1, TEMP tmp2);
+ins_cost(300);
+format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   // TEMP $tmp1, $tmp2" %}
+ins_encode( enc_String_Compare(str1, str2, cnt1, cnt2, result, tmp1, tmp2) );
+ins_pipe(long_memory_op);
+%}
+// FIXME: Why R0/R1/R2?
+instruct string_equals(R0RegP str1, R1RegP str2, R2RegI cnt, iRegI result, iRegI tmp1, iRegI tmp2,
+flagsReg ccr) %{
+predicate(!CompactStrings);
+match(Set result (StrEquals (Binary str1 str2) cnt));
+effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, TEMP tmp1, TEMP tmp2, TEMP result, KILL ccr);
+ins_cost(300);
+format %{ "String Equals $str1,$str2,$cnt -> $result   // TEMP $tmp1, $tmp2" %}
+ins_encode( enc_String_Equals(str1, str2, cnt, result, tmp1, tmp2) );
+ins_pipe(long_memory_op);
+%}
+// FIXME: Why R0/R1?
+instruct array_equals(R0RegP ary1, R1RegP ary2, iRegI tmp1, iRegI tmp2, iRegI tmp3, iRegI result,
+flagsReg ccr) %{
+predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
+match(Set result (AryEq ary1 ary2));
+effect(USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP result, KILL ccr);
+ins_cost(300);
+format %{ "Array Equals $ary1,$ary2 -> $result   // TEMP $tmp1,$tmp2,$tmp3" %}
+ins_encode( enc_Array_Equals(ary1, ary2, tmp1, tmp2, tmp3, result));
+ins_pipe(long_memory_op);
+%}
+#endif
+//---------- Zeros Count Instructions ------------------------------------------
+instruct countLeadingZerosI(iRegI dst, iRegI src) %{
+match(Set dst (CountLeadingZerosI src));
+size(4);
+format %{ "CLZ_32 $dst,$src" %}
+ins_encode %{
+__ clz_32($dst$$Register, $src$$Register);
+%}
+ins_pipe(ialu_reg);
+%}
+#ifdef AARCH64
+instruct countLeadingZerosL(iRegI dst, iRegL src) %{
+match(Set dst (CountLeadingZerosL src));
+size(4);
+format %{ "CLZ $dst,$src" %}
+ins_encode %{
+__ clz($dst$$Register, $src$$Register);
+%}
+ins_pipe(ialu_reg);
+%}
+#else
+instruct countLeadingZerosL(iRegI dst, iRegL src, iRegI tmp, flagsReg ccr) %{
+match(Set dst (CountLeadingZerosL src));
+effect(TEMP tmp, TEMP dst, KILL ccr);
+size(16);
+format %{ "CLZ    $dst,$src.hi\n\t"
+"TEQ    $dst,32\n\t"
+"CLZ.eq $tmp,$src.lo\n\t"
+"ADD.eq $dst, $dst, $tmp\n\t" %}
+ins_encode %{
+__ clz($dst$$Register, $src$$Register->successor());
+__ teq($dst$$Register, 32);
+__ clz($tmp$$Register, $src$$Register, eq);
+__ add($dst$$Register, $dst$$Register, $tmp$$Register, eq);
+%}
+ins_pipe(ialu_reg);
+%}
+#endif
+instruct countTrailingZerosI(iRegI dst, iRegI src, iRegI tmp) %{
+match(Set dst (CountTrailingZerosI src));
+effect(TEMP tmp);
+size(8);
+format %{ "RBIT_32 $tmp, $src\n\t"
+"CLZ_32  $dst,$tmp" %}
+ins_encode %{
+__ rbit_32($tmp$$Register, $src$$Register);
+__ clz_32($dst$$Register, $tmp$$Register);
+%}
+ins_pipe(ialu_reg);
+%}
+#ifdef AARCH64
+instruct countTrailingZerosL(iRegI dst, iRegL src, iRegL tmp) %{
+match(Set dst (CountTrailingZerosL src));
+effect(TEMP tmp);
+size(8);
+format %{ "RBIT $tmp, $src\n\t"
+"CLZ  $dst,$tmp" %}
+ins_encode %{
+__ rbit($tmp$$Register, $src$$Register);
+__ clz($dst$$Register, $tmp$$Register);
+%}
+ins_pipe(ialu_reg);
+%}
+#else
+instruct countTrailingZerosL(iRegI dst, iRegL src, iRegI tmp, flagsReg ccr) %{
+match(Set dst (CountTrailingZerosL src));
+effect(TEMP tmp, TEMP dst, KILL ccr);
+size(24);
+format %{ "RBIT   $tmp,$src.lo\n\t"
+"CLZ    $dst,$tmp\n\t"
+"TEQ    $dst,32\n\t"
+"RBIT   $tmp,$src.hi\n\t"
+"CLZ.eq $tmp,$tmp\n\t"
+"ADD.eq $dst,$dst,$tmp\n\t" %}
+ins_encode %{
+__ rbit($tmp$$Register, $src$$Register);
+__ clz($dst$$Register, $tmp$$Register);
+__ teq($dst$$Register, 32);
+__ rbit($tmp$$Register, $src$$Register->successor());
+__ clz($tmp$$Register, $tmp$$Register, eq);
+__ add($dst$$Register, $dst$$Register, $tmp$$Register, eq);
+%}
+ins_pipe(ialu_reg);
+%}
+#endif
+//---------- Population Count Instructions -------------------------------------
+#ifdef AARCH64
+instruct popCountI(iRegI dst, iRegI src, regD_low tmp) %{
+predicate(UsePopCountInstruction);
+match(Set dst (PopCountI src));
+effect(TEMP tmp);
+size(20);
+format %{ "MOV_W      $dst,$src\n\t"
+"FMOV_dx    $tmp,$dst\n\t"
+"VCNT       $tmp.8B,$tmp.8B\n\t"
+"ADDV       $tmp.B,$tmp.8B\n\t"
+"FMRS       $dst,$tmp" %}
+ins_encode %{
+__ mov_w($dst$$Register, $src$$Register);
+__ fmov_dx($tmp$$FloatRegister, $dst$$Register);
+int quad = 0;
+int cnt_size = 0; // VELEM_SIZE_8
+__ vcnt($tmp$$FloatRegister, $tmp$$FloatRegister, quad, cnt_size);
+int add_size = 0; // VELEM_SIZE_8
+__ addv($tmp$$FloatRegister, $tmp$$FloatRegister, quad, add_size);
+__ fmrs($dst$$Register, $tmp$$FloatRegister);
+%}
+ins_pipe(ialu_reg); // FIXME
+%}
+#else
+instruct popCountI(iRegI dst, iRegI src, regD_low tmp) %{
+predicate(UsePopCountInstruction);
+match(Set dst (PopCountI src));
+effect(TEMP tmp);
+format %{ "FMSR       $tmp,$src\n\t"
+"VCNT.8     $tmp,$tmp\n\t"
+"VPADDL.U8  $tmp,$tmp\n\t"
+"VPADDL.U16 $tmp,$tmp\n\t"
+"FMRS       $dst,$tmp" %}
+size(20);
+ins_encode %{
+__ fmsr($tmp$$FloatRegister, $src$$Register);
+__ vcnt($tmp$$FloatRegister, $tmp$$FloatRegister);
+__ vpaddl($tmp$$FloatRegister, $tmp$$FloatRegister, 8, 0);
+__ vpaddl($tmp$$FloatRegister, $tmp$$FloatRegister, 16, 0);
+__ fmrs($dst$$Register, $tmp$$FloatRegister);
+%}
+ins_pipe(ialu_reg); // FIXME
+%}
+#endif
+#ifdef AARCH64
+instruct popCountL(iRegI dst, iRegL src, regD tmp) %{
+predicate(UsePopCountInstruction);
+match(Set dst (PopCountL src));
+effect(TEMP tmp);
+size(16);
+format %{ "FMOV_dx    $tmp,$src\n\t"
+"VCNT       $tmp.8B,$tmp.8B\n\t"
+"ADDV       $tmp.B,$tmp.8B\n\t"
+"FMOV_ws    $dst,$tmp" %}
+ins_encode %{
+__ fmov_dx($tmp$$FloatRegister, $src$$Register);
+int quad = 0;
+int cnt_size = 0;
+__ vcnt($tmp$$FloatRegister, $tmp$$FloatRegister, quad, cnt_size);
+int add_size = 0;
+__ addv($tmp$$FloatRegister, $tmp$$FloatRegister, quad, add_size);
+__ fmov_ws($dst$$Register, $tmp$$FloatRegister);
+%}
+ins_pipe(ialu_reg); // FIXME
+%}
+#else
+// Note: Long.bitCount(long) returns an int.
+instruct popCountL(iRegI dst, iRegL src, regD_low tmp) %{
+predicate(UsePopCountInstruction);
+match(Set dst (PopCountL src));
+effect(TEMP tmp);
+format %{ "FMDRR       $tmp,$src.lo,$src.hi\n\t"
+"VCNT.8      $tmp,$tmp\n\t"
+"VPADDL.U8   $tmp,$tmp\n\t"
+"VPADDL.U16  $tmp,$tmp\n\t"
+"VPADDL.U32  $tmp,$tmp\n\t"
+"FMRS        $dst,$tmp" %}
+size(32);
+ins_encode %{
+__ fmdrr($tmp$$FloatRegister, $src$$Register, $src$$Register->successor());
+__ vcnt($tmp$$FloatRegister, $tmp$$FloatRegister);
+__ vpaddl($tmp$$FloatRegister, $tmp$$FloatRegister, 8, 0);
+__ vpaddl($tmp$$FloatRegister, $tmp$$FloatRegister, 16, 0);
+__ vpaddl($tmp$$FloatRegister, $tmp$$FloatRegister, 32, 0);
+__ fmrs($dst$$Register, $tmp$$FloatRegister);
+%}
+ins_pipe(ialu_reg);
+%}
+#endif
+// ============================================================================
+//------------Bytes reverse--------------------------------------------------
+instruct bytes_reverse_int(iRegI dst, iRegI src) %{
+match(Set dst (ReverseBytesI src));
+size(4);
+format %{ "REV32 $dst,$src" %}
+ins_encode %{
+#ifdef AARCH64
+__ rev_w($dst$$Register, $src$$Register);
+// high 32 bits zeroed, not sign extended
+#else
+__ rev($dst$$Register, $src$$Register);
+#endif
+%}
+ins_pipe( iload_mem ); // FIXME
+%}
+instruct bytes_reverse_long(iRegL dst, iRegL src) %{
+match(Set dst (ReverseBytesL src));
+#ifdef AARCH64
+//size(4);
+format %{ "REV $dst,$src"  %}
+ins_encode %{
+__ rev($dst$$Register, $src$$Register);
+%}
+ins_pipe(ialu_reg_reg); // FIXME
+#else
+effect(TEMP dst);
+size(8);
+format %{ "REV $dst.lo,$src.lo\n\t"
+"REV $dst.hi,$src.hi" %}
+ins_encode %{
+__ rev($dst$$Register, $src$$Register->successor());
+__ rev($dst$$Register->successor(), $src$$Register);
+%}
+ins_pipe( iload_mem ); // FIXME
+#endif
+%}
+instruct bytes_reverse_unsigned_short(iRegI dst, iRegI src) %{
+match(Set dst (ReverseBytesUS src));
+#ifdef AARCH64
+size(4);
+format %{ "REV16_W $dst,$src" %}
+ins_encode %{
+__ rev16_w($dst$$Register, $src$$Register);
+// high 32 bits zeroed
+%}
+#else
+size(4);
+format %{ "REV16 $dst,$src" %}
+ins_encode %{
+__ rev16($dst$$Register, $src$$Register);
+%}
+#endif
+ins_pipe( iload_mem ); // FIXME
+%}
+instruct bytes_reverse_short(iRegI dst, iRegI src) %{
+match(Set dst (ReverseBytesS src));
+#ifdef AARCH64
+size(8);
+format %{ "REV16_W $dst,$src\n\t"
+"SIGN_EXT16 $dst" %}
+ins_encode %{
+__ rev16_w($dst$$Register, $src$$Register);
+__ sign_extend($dst$$Register, $dst$$Register, 16);
+%}
+#else
+size(4);
+format %{ "REVSH $dst,$src" %}
+ins_encode %{
+__ revsh($dst$$Register, $src$$Register);
+%}
+#endif
+ins_pipe( iload_mem ); // FIXME
+%}
+// ====================VECTOR INSTRUCTIONS=====================================
+// Load Aligned Packed values into a Double Register
+instruct loadV8(vecD dst, memoryD mem) %{
+predicate(n->as_LoadVector()->memory_size() == 8);
+match(Set dst (LoadVector mem));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "FLDD   $mem,$dst\t! load vector (8 bytes)" %}
+ins_encode %{
+__ ldr_double($dst$$FloatRegister, $mem$$Address);
+%}
+ins_pipe(floadD_mem);
+%}
+// Load Aligned Packed values into a Double Register Pair
+instruct loadV16(vecX dst, memoryvld mem) %{
+predicate(n->as_LoadVector()->memory_size() == 16);
+match(Set dst (LoadVector mem));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "VLD1   $mem,$dst.Q\t! load vector (16 bytes)" %}
+ins_encode %{
+__ vld1($dst$$FloatRegister, $mem$$Address, MacroAssembler::VELEM_SIZE_16, 128);
+%}
+ins_pipe(floadD_mem); // FIXME
+%}
+// Store Vector in Double register to memory
+instruct storeV8(memoryD mem, vecD src) %{
+predicate(n->as_StoreVector()->memory_size() == 8);
+match(Set mem (StoreVector mem src));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "FSTD   $src,$mem\t! store vector (8 bytes)" %}
+ins_encode %{
+__ str_double($src$$FloatRegister, $mem$$Address);
+%}
+ins_pipe(fstoreD_mem_reg);
+%}
+// Store Vector in Double Register Pair to memory
+instruct storeV16(memoryvld mem, vecX src) %{
+predicate(n->as_StoreVector()->memory_size() == 16);
+match(Set mem (StoreVector mem src));
+ins_cost(MEMORY_REF_COST);
+size(4);
+format %{ "VST1   $src,$mem\t! store vector (16 bytes)" %}
+ins_encode %{
+__ vst1($src$$FloatRegister, $mem$$Address, MacroAssembler::VELEM_SIZE_16, 128);
+%}
+ins_pipe(fstoreD_mem_reg); // FIXME
+%}
+#ifndef AARCH64
+// Replicate scalar to packed byte values in Double register
+instruct Repl8B_reg(vecD dst, iRegI src, iRegI tmp) %{
+predicate(n->as_Vector()->length() == 8);
+match(Set dst (ReplicateB src));
+ins_cost(DEFAULT_COST*4);
+effect(TEMP tmp);
+size(16);
+// FIXME: could use PKH instruction instead?
+format %{ "LSL      $tmp, $src, 24 \n\t"
+"OR       $tmp, $tmp, ($tmp >> 8) \n\t"
+"OR       $tmp, $tmp, ($tmp >> 16) \n\t"
+"FMDRR    $dst,$tmp,$tmp\t" %}
+ins_encode %{
+__ mov($tmp$$Register, AsmOperand($src$$Register, lsl, 24));
+__ orr($tmp$$Register, $tmp$$Register, AsmOperand($tmp$$Register, lsr, 8));
+__ orr($tmp$$Register, $tmp$$Register, AsmOperand($tmp$$Register, lsr, 16));
+__ fmdrr($dst$$FloatRegister, $tmp$$Register, $tmp$$Register);
+%}
+ins_pipe(ialu_reg); // FIXME
+%}
+#endif /* !AARCH64 */
+// Replicate scalar to packed byte values in Double register
+instruct Repl8B_reg_simd(vecD dst, iRegI src) %{
+predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
+match(Set dst (ReplicateB src));
+size(4);
+format %{ "VDUP.8 $dst,$src\t" %}
+ins_encode %{
+bool quad = false;
+__ vdupI($dst$$FloatRegister, $src$$Register,
+MacroAssembler::VELEM_SIZE_8, quad);
+%}
+ins_pipe(ialu_reg); // FIXME
+%}
+// Replicate scalar to packed byte values in Double register pair
+instruct Repl16B_reg(vecX dst, iRegI src) %{
+predicate(n->as_Vector()->length_in_bytes() == 16);
+match(Set dst (ReplicateB src));
+size(4);
+format %{ "VDUP.8 $dst.Q,$src\t" %}
+ins_encode %{
+bool quad = true;
+__ vdupI($dst$$FloatRegister, $src$$Register,
+MacroAssembler::VELEM_SIZE_8, quad);
+%}
+ins_pipe(ialu_reg); // FIXME
+%}
+#ifndef AARCH64
+// Replicate scalar constant to packed byte values in Double register
+instruct Repl8B_immI(vecD dst, immI src, iRegI tmp) %{
+predicate(n->as_Vector()->length() == 8);
+match(Set dst (ReplicateB src));
+ins_cost(DEFAULT_COST*2);
+effect(TEMP tmp);
+size(12);
+format %{ "MOV      $tmp, Repl4($src))\n\t"
+"FMDRR    $dst,$tmp,$tmp\t" %}
+ins_encode( LdReplImmI(src, dst, tmp, (4), (1)) );
+ins_pipe(loadConFD); // FIXME
+%}
+#endif /* !AARCH64 */
+// Replicate scalar constant to packed byte values in Double register
+// TODO: support negative constants with MVNI?
+instruct Repl8B_immU8(vecD dst, immU8 src) %{
+predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
+match(Set dst (ReplicateB src));
+size(4);
+format %{ "VMOV.U8  $dst,$src" %}
+ins_encode %{
+bool quad = false;
+__ vmovI($dst$$FloatRegister, $src$$constant,
+MacroAssembler::VELEM_SIZE_8, quad);
+%}
+ins_pipe(loadConFD); // FIXME
+%}
+// Replicate scalar constant to packed byte values in Double register pair
+instruct Repl16B_immU8(vecX dst, immU8 src) %{
+predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
+match(Set dst (ReplicateB src));
+size(4);
+format %{ "VMOV.U8  $dst.Q,$src" %}
+ins_encode %{
+bool quad = true;
+__ vmovI($dst$$FloatRegister, $src$$constant,
+MacroAssembler::VELEM_SIZE_8, quad);
+%}
+ins_pipe(loadConFD); // FIXME
+%}
+#ifndef AARCH64
+// Replicate scalar to packed short/char values into Double register
+instruct Repl4S_reg(vecD dst, iRegI src, iRegI tmp) %{
+predicate(n->as_Vector()->length() == 4);
+match(Set dst (ReplicateS src));
+ins_cost(DEFAULT_COST*3);
+effect(TEMP tmp);
+size(12);
+// FIXME: could use PKH instruction instead?
+format %{ "LSL      $tmp, $src, 16 \n\t"
+"OR       $tmp, $tmp, ($tmp >> 16) \n\t"
+"FMDRR    $dst,$tmp,$tmp\t" %}
+ins_encode %{
+__ mov($tmp$$Register, AsmOperand($src$$Register, lsl, 16));
+__ orr($tmp$$Register, $tmp$$Register, AsmOperand($tmp$$Register, lsr, 16));
+__ fmdrr($dst$$FloatRegister, $tmp$$Register, $tmp$$Register);
+%}
+ins_pipe(ialu_reg); // FIXME
+%}
+#endif /* !AARCH64 */
+// Replicate scalar to packed byte values in Double register
+instruct Repl4S_reg_simd(vecD dst, iRegI src) %{
+predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
+match(Set dst (ReplicateS src));
+size(4);
+format %{ "VDUP.16 $dst,$src\t" %}
+ins_encode %{
+bool quad = false;
+__ vdupI($dst$$FloatRegister, $src$$Register,
+MacroAssembler::VELEM_SIZE_16, quad);
+%}
+ins_pipe(ialu_reg); // FIXME
+%}
+// Replicate scalar to packed byte values in Double register pair
+instruct Repl8S_reg(vecX dst, iRegI src) %{
+predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
+match(Set dst (ReplicateS src));
+size(4);
+format %{ "VDUP.16 $dst.Q,$src\t" %}
+ins_encode %{
+bool quad = true;
+__ vdupI($dst$$FloatRegister, $src$$Register,
+MacroAssembler::VELEM_SIZE_16, quad);
+%}
+ins_pipe(ialu_reg); // FIXME
+%}
+#ifndef AARCH64
+// Replicate scalar constant to packed short/char values in Double register
+instruct Repl4S_immI(vecD dst, immI src, iRegP tmp) %{
+predicate(n->as_Vector()->length() == 4);
+match(Set dst (ReplicateS src));
+effect(TEMP tmp);
+size(12);
+ins_cost(DEFAULT_COST*4); // FIXME
+format %{ "MOV      $tmp, Repl2($src))\n\t"
+"FMDRR    $dst,$tmp,$tmp\t" %}
+ins_encode( LdReplImmI(src, dst, tmp, (2), (2)) );
+ins_pipe(loadConFD); // FIXME
+%}
+#endif /* !AARCH64 */
+// Replicate scalar constant to packed byte values in Double register
+instruct Repl4S_immU8(vecD dst, immU8 src) %{
+predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
+match(Set dst (ReplicateS src));
+size(4);
+format %{ "VMOV.U16  $dst,$src" %}
+ins_encode %{
+bool quad = false;
+__ vmovI($dst$$FloatRegister, $src$$constant,
+MacroAssembler::VELEM_SIZE_16, quad);
+%}
+ins_pipe(loadConFD); // FIXME
+%}
+// Replicate scalar constant to packed byte values in Double register pair
+instruct Repl8S_immU8(vecX dst, immU8 src) %{
+predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
+match(Set dst (ReplicateS src));
+size(4);
+format %{ "VMOV.U16  $dst.Q,$src" %}
+ins_encode %{
+bool quad = true;
+__ vmovI($dst$$FloatRegister, $src$$constant,
+MacroAssembler::VELEM_SIZE_16, quad);
+%}
+ins_pipe(loadConFD); // FIXME
+%}
+#ifndef AARCH64
+// Replicate scalar to packed int values in Double register
+instruct Repl2I_reg(vecD dst, iRegI src) %{
+predicate(n->as_Vector()->length() == 2);
+match(Set dst (ReplicateI src));
+size(4);
+format %{ "FMDRR    $dst,$src,$src\t" %}
+ins_encode %{
+__ fmdrr($dst$$FloatRegister, $src$$Register, $src$$Register);
+%}
+ins_pipe(ialu_reg); // FIXME
+%}
+// Replicate scalar to packed int values in Double register pair
+instruct Repl4I_reg(vecX dst, iRegI src) %{
+predicate(n->as_Vector()->length() == 4);
+match(Set dst (ReplicateI src));
+ins_cost(DEFAULT_COST*2);
+size(8);
+format %{ "FMDRR    $dst.lo,$src,$src\n\t"
+"FMDRR    $dst.hi,$src,$src" %}
+ins_encode %{
+__ fmdrr($dst$$FloatRegister, $src$$Register, $src$$Register);
+__ fmdrr($dst$$FloatRegister->successor()->successor(),
+$src$$Register, $src$$Register);
+%}
+ins_pipe(ialu_reg); // FIXME
+%}
+#endif /* !AARCH64 */
+// Replicate scalar to packed int values in Double register
+instruct Repl2I_reg_simd(vecD dst, iRegI src) %{
+predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
+match(Set dst (ReplicateI src));
+size(4);
+format %{ "VDUP.32 $dst.D,$src\t" %}
+ins_encode %{
+bool quad = false;
+__ vdupI($dst$$FloatRegister, $src$$Register,
+MacroAssembler::VELEM_SIZE_32, quad);
+%}
+ins_pipe(ialu_reg); // FIXME
+%}
+// Replicate scalar to packed int values in Double register pair
+instruct Repl4I_reg_simd(vecX dst, iRegI src) %{
+predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
+match(Set dst (ReplicateI src));
+size(4);
+format %{ "VDUP.32 $dst.Q,$src\t" %}
+ins_encode %{
+bool quad = true;
+__ vdupI($dst$$FloatRegister, $src$$Register,
+MacroAssembler::VELEM_SIZE_32, quad);
+%}
+ins_pipe(ialu_reg); // FIXME
+%}
+#ifndef AARCH64
+// Replicate scalar zero constant to packed int values in Double register
+instruct Repl2I_immI(vecD dst, immI src, iRegI tmp) %{
+predicate(n->as_Vector()->length() == 2);
+match(Set dst (ReplicateI src));
+effect(TEMP tmp);
+size(12);
+ins_cost(DEFAULT_COST*4); // FIXME
+format %{ "MOV      $tmp, Repl1($src))\n\t"
+"FMDRR    $dst,$tmp,$tmp\t" %}
+ins_encode( LdReplImmI(src, dst, tmp, (1), (4)) );
+ins_pipe(loadConFD); // FIXME
+%}
+#endif /* !AARCH64 */
+// Replicate scalar constant to packed byte values in Double register
+instruct Repl2I_immU8(vecD dst, immU8 src) %{
+predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
+match(Set dst (ReplicateI src));
+size(4);
+format %{ "VMOV.I32  $dst.D,$src" %}
+ins_encode %{
+bool quad = false;
+__ vmovI($dst$$FloatRegister, $src$$constant,
+MacroAssembler::VELEM_SIZE_32, quad);
+%}
+ins_pipe(loadConFD); // FIXME
+%}
+// Replicate scalar constant to packed byte values in Double register pair
+instruct Repl4I_immU8(vecX dst, immU8 src) %{
+predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
+match(Set dst (ReplicateI src));
+size(4);
+format %{ "VMOV.I32  $dst.Q,$src" %}
+ins_encode %{
+bool quad = true;
+__ vmovI($dst$$FloatRegister, $src$$constant,
+MacroAssembler::VELEM_SIZE_32, quad);
+%}
+ins_pipe(loadConFD); // FIXME
+%}
+#ifdef AARCH64
+// Replicate scalar to packed byte values in Double register pair
+instruct Repl2L_reg(vecX dst, iRegL src) %{
+predicate(n->as_Vector()->length() == 2);
+match(Set dst (ReplicateL src));
+size(4*1);
+ins_cost(DEFAULT_COST*1); // FIXME
+format %{ "VDUP.2D $dst.Q,$src\t" %}
+ins_encode %{
+bool quad = true;
+__ vdupI($dst$$FloatRegister, $src$$Register,
+MacroAssembler::VELEM_SIZE_64, quad);
+%}
+ins_pipe(ialu_reg); // FIXME
+%}
+#else /* !AARCH64 */
+// Replicate scalar to packed byte values in Double register pair
+instruct Repl2L_reg(vecX dst, iRegL src) %{
+predicate(n->as_Vector()->length() == 2);
+match(Set dst (ReplicateL src));
+size(8);
+ins_cost(DEFAULT_COST*2); // FIXME
+format %{ "FMDRR $dst.D,$src.lo,$src.hi\t\n"
+"FMDRR $dst.D.next,$src.lo,$src.hi" %}
+ins_encode %{
+__ fmdrr($dst$$FloatRegister, $src$$Register, $src$$Register->successor());
+__ fmdrr($dst$$FloatRegister->successor()->successor(),
+$src$$Register, $src$$Register->successor());
+%}
+ins_pipe(ialu_reg); // FIXME
+%}
+// Replicate scalar to packed float values in Double register
+instruct Repl2F_regI(vecD dst, iRegI src) %{
+predicate(n->as_Vector()->length() == 2);
+match(Set dst (ReplicateF src));
+size(4);
+format %{ "FMDRR    $dst.D,$src,$src\t" %}
+ins_encode %{
+__ fmdrr($dst$$FloatRegister, $src$$Register, $src$$Register);
+%}
+ins_pipe(ialu_reg); // FIXME
+%}
+// Replicate scalar to packed float values in Double register
+instruct Repl2F_reg_vfp(vecD dst, regF src) %{
+predicate(n->as_Vector()->length() == 2);
+match(Set dst (ReplicateF src));
+size(4*2);
+ins_cost(DEFAULT_COST*2); // FIXME
+expand %{
+iRegI tmp;
+MoveF2I_reg_reg(tmp, src);
+Repl2F_regI(dst,tmp);
+%}
+%}
+#endif /* !AARCH64 */
+// Replicate scalar to packed float values in Double register
+instruct Repl2F_reg_simd(vecD dst, regF src) %{
+predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
+match(Set dst (ReplicateF src));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{ "VDUP.32  $dst.D,$src.D\t" %}
+ins_encode %{
+bool quad = false;
+__ vdupF($dst$$FloatRegister, $src$$FloatRegister, quad);
+%}
+ins_pipe(ialu_reg); // FIXME
+%}
+#ifndef AARCH64
+// Replicate scalar to packed float values in Double register pair
+instruct Repl4F_reg(vecX dst, regF src, iRegI tmp) %{
+predicate(n->as_Vector()->length() == 4);
+match(Set dst (ReplicateF src));
+effect(TEMP tmp);
+size(4*3);
+ins_cost(DEFAULT_COST*3); // FIXME
+format %{ "FMRS     $tmp,$src\n\t"
+"FMDRR    $dst.D,$tmp,$tmp\n\t"
+"FMDRR    $dst.D.next,$tmp,$tmp\t" %}
+ins_encode %{
+__ fmrs($tmp$$Register, $src$$FloatRegister);
+__ fmdrr($dst$$FloatRegister, $tmp$$Register, $tmp$$Register);
+__ fmdrr($dst$$FloatRegister->successor()->successor(),
+$tmp$$Register, $tmp$$Register);
+%}
+ins_pipe(ialu_reg); // FIXME
+%}
+#endif /* !AARCH64 */
+// Replicate scalar to packed float values in Double register pair
+instruct Repl4F_reg_simd(vecX dst, regF src) %{
+predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
+match(Set dst (ReplicateF src));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{ "VDUP.32  $dst.Q,$src.D\t" %}
+ins_encode %{
+bool quad = true;
+__ vdupF($dst$$FloatRegister, $src$$FloatRegister, quad);
+%}
+ins_pipe(ialu_reg); // FIXME
+%}
+#ifndef AARCH64
+// Replicate scalar zero constant to packed float values in Double register
+instruct Repl2F_immI(vecD dst, immF src, iRegI tmp) %{
+predicate(n->as_Vector()->length() == 2);
+match(Set dst (ReplicateF src));
+effect(TEMP tmp);
+size(12);
+ins_cost(DEFAULT_COST*4); // FIXME
+format %{ "MOV      $tmp, Repl1($src))\n\t"
+"FMDRR    $dst,$tmp,$tmp\t" %}
+ins_encode( LdReplImmF(src, dst, tmp) );
+ins_pipe(loadConFD); // FIXME
+%}
+#endif /* !AAARCH64 */
+// Replicate scalar to packed double float values in Double register pair
+instruct Repl2D_reg(vecX dst, regD src) %{
+#ifdef AARCH64
+predicate(n->as_Vector()->length() == 2 && VM_Version::has_simd());
+match(Set dst (ReplicateD src));
+size(4*1);
+ins_cost(DEFAULT_COST*1); // FIXME
+format %{ "VDUP     $dst.2D,$src\t" %}
+ins_encode %{
+bool quad = true;
+__ vdupD($dst$$FloatRegister, $src$$FloatRegister, quad);
+%}
+#else
+predicate(n->as_Vector()->length() == 2);
+match(Set dst (ReplicateD src));
+size(4*2);
+ins_cost(DEFAULT_COST*2); // FIXME
+format %{ "FCPYD    $dst.D.a,$src\n\t"
+"FCPYD    $dst.D.b,$src\t" %}
+ins_encode %{
+FloatRegister dsta = $dst$$FloatRegister;
+FloatRegister src = $src$$FloatRegister;
+__ fcpyd(dsta, src);
+FloatRegister dstb = dsta->successor()->successor();
+__ fcpyd(dstb, src);
+%}
+#endif
+ins_pipe(ialu_reg); // FIXME
+%}
+// ====================VECTOR ARITHMETIC=======================================
+// --------------------------------- ADD --------------------------------------
+// Bytes vector add
+instruct vadd8B_reg(vecD dst, vecD src1, vecD src2) %{
+predicate(n->as_Vector()->length() == 8);
+match(Set dst (AddVB src1 src2));
+format %{ "VADD.I8 $dst,$src1,$src2\t! add packed8B" %}
+size(4);
+ins_encode %{
+bool quad = false;
+__ vaddI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VELEM_SIZE_8, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+instruct vadd16B_reg(vecX dst, vecX src1, vecX src2) %{
+predicate(n->as_Vector()->length() == 16);
+match(Set dst (AddVB src1 src2));
+size(4);
+format %{ "VADD.I8 $dst.Q,$src1.Q,$src2.Q\t! add packed16B" %}
+ins_encode %{
+bool quad = true;
+__ vaddI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VELEM_SIZE_8, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// Shorts/Chars vector add
+instruct vadd4S_reg(vecD dst, vecD src1, vecD src2) %{
+predicate(n->as_Vector()->length() == 4);
+match(Set dst (AddVS src1 src2));
+size(4);
+format %{ "VADD.I16 $dst,$src1,$src2\t! add packed4S" %}
+ins_encode %{
+bool quad = false;
+__ vaddI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VELEM_SIZE_16, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+instruct vadd8S_reg(vecX dst, vecX src1, vecX src2) %{
+predicate(n->as_Vector()->length() == 8);
+match(Set dst (AddVS src1 src2));
+size(4);
+format %{ "VADD.I16 $dst.Q,$src1.Q,$src2.Q\t! add packed8S" %}
+ins_encode %{
+bool quad = true;
+__ vaddI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VELEM_SIZE_16, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// Integers vector add
+instruct vadd2I_reg(vecD dst, vecD src1, vecD src2) %{
+predicate(n->as_Vector()->length() == 2);
+match(Set dst (AddVI src1 src2));
+size(4);
+format %{ "VADD.I32 $dst.D,$src1.D,$src2.D\t! add packed2I" %}
+ins_encode %{
+bool quad = false;
+__ vaddI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VELEM_SIZE_32, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+instruct vadd4I_reg(vecX dst, vecX src1, vecX src2) %{
+predicate(n->as_Vector()->length() == 4);
+match(Set dst (AddVI src1 src2));
+size(4);
+format %{ "VADD.I32 $dst.Q,$src1.Q,$src2.Q\t! add packed4I" %}
+ins_encode %{
+bool quad = true;
+__ vaddI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VELEM_SIZE_32, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// Longs vector add
+instruct vadd2L_reg(vecX dst, vecX src1, vecX src2) %{
+predicate(n->as_Vector()->length() == 2);
+match(Set dst (AddVL src1 src2));
+size(4);
+format %{ "VADD.I64 $dst.Q,$src1.Q,$src2.Q\t! add packed2L" %}
+ins_encode %{
+bool quad = true;
+__ vaddI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VELEM_SIZE_64, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// Floats vector add
+instruct vadd2F_reg(vecD dst, vecD src1, vecD src2) %{
+predicate(n->as_Vector()->length() == 2 && VM_Version::simd_math_is_compliant());
+match(Set dst (AddVF src1 src2));
+size(4);
+format %{ "VADD.F32 $dst,$src1,$src2\t! add packed2F" %}
+ins_encode %{
+bool quad = false;
+__ vaddF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VFA_SIZE_F32, quad);
+%}
+ins_pipe( faddD_reg_reg ); // FIXME
+%}
+#ifndef AARCH64
+instruct vadd2F_reg_vfp(vecD dst, vecD src1, vecD src2) %{
+predicate(n->as_Vector()->length() == 2 && !VM_Version::simd_math_is_compliant());
+match(Set dst (AddVF src1 src2));
+ins_cost(DEFAULT_COST*2); // FIXME
+size(4*2);
+format %{ "FADDS  $dst.a,$src1.a,$src2.a\n\t"
+"FADDS  $dst.b,$src1.b,$src2.b" %}
+ins_encode %{
+__ add_float($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
+__ add_float($dst$$FloatRegister->successor(),
+$src1$$FloatRegister->successor(),
+$src2$$FloatRegister->successor());
+%}
+ins_pipe(faddF_reg_reg); // FIXME
+%}
+#endif
+instruct vadd4F_reg_simd(vecX dst, vecX src1, vecX src2) %{
+predicate(n->as_Vector()->length() == 4 && VM_Version::simd_math_is_compliant());
+match(Set dst (AddVF src1 src2));
+size(4);
+format %{ "VADD.F32 $dst.Q,$src1.Q,$src2.Q\t! add packed4F" %}
+ins_encode %{
+bool quad = true;
+__ vaddF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VFA_SIZE_F32, quad);
+%}
+ins_pipe( faddD_reg_reg ); // FIXME
+%}
+#ifdef AARCH64
+instruct vadd2D_reg_simd(vecX dst, vecX src1, vecX src2) %{
+predicate(n->as_Vector()->length() == 2 && VM_Version::simd_math_is_compliant());
+match(Set dst (AddVD src1 src2));
+size(4);
+format %{ "VADD.F64 $dst.Q,$src1.Q,$src2.Q\t! add packed2D" %}
+ins_encode %{
+bool quad = true;
+__ vaddF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VFA_SIZE_F64, quad);
+%}
+ins_pipe( faddD_reg_reg ); // FIXME
+%}
+#else
+instruct vadd4F_reg_vfp(vecX dst, vecX src1, vecX src2) %{
+predicate(n->as_Vector()->length() == 4 && !VM_Version::simd_math_is_compliant());
+match(Set dst (AddVF src1 src2));
+size(4*4);
+ins_cost(DEFAULT_COST*4); // FIXME
+format %{ "FADDS  $dst.a,$src1.a,$src2.a\n\t"
+"FADDS  $dst.b,$src1.b,$src2.b\n\t"
+"FADDS  $dst.c,$src1.c,$src2.c\n\t"
+"FADDS  $dst.d,$src1.d,$src2.d" %}
+ins_encode %{
+FloatRegister dsta = $dst$$FloatRegister;
+FloatRegister src1a = $src1$$FloatRegister;
+FloatRegister src2a = $src2$$FloatRegister;
+__ add_float(dsta, src1a, src2a);
+FloatRegister dstb = dsta->successor();
+FloatRegister src1b = src1a->successor();
+FloatRegister src2b = src2a->successor();
+__ add_float(dstb, src1b, src2b);
+FloatRegister dstc = dstb->successor();
+FloatRegister src1c = src1b->successor();
+FloatRegister src2c = src2b->successor();
+__ add_float(dstc, src1c, src2c);
+FloatRegister dstd = dstc->successor();
+FloatRegister src1d = src1c->successor();
+FloatRegister src2d = src2c->successor();
+__ add_float(dstd, src1d, src2d);
+%}
+ins_pipe(faddF_reg_reg); // FIXME
+%}
+instruct vadd2D_reg_vfp(vecX dst, vecX src1, vecX src2) %{
+predicate(n->as_Vector()->length() == 2);
+match(Set dst (AddVD src1 src2));
+size(4*2);
+ins_cost(DEFAULT_COST*2); // FIXME
+format %{ "FADDD  $dst.a,$src1.a,$src2.a\n\t"
+"FADDD  $dst.b,$src1.b,$src2.b" %}
+ins_encode %{
+FloatRegister dsta = $dst$$FloatRegister;
+FloatRegister src1a = $src1$$FloatRegister;
+FloatRegister src2a = $src2$$FloatRegister;
+__ add_double(dsta, src1a, src2a);
+FloatRegister dstb = dsta->successor()->successor();
+FloatRegister src1b = src1a->successor()->successor();
+FloatRegister src2b = src2a->successor()->successor();
+__ add_double(dstb, src1b, src2b);
+%}
+ins_pipe(faddF_reg_reg); // FIXME
+%}
+#endif
+// Bytes vector sub
+instruct vsub8B_reg(vecD dst, vecD src1, vecD src2) %{
+predicate(n->as_Vector()->length() == 8);
+match(Set dst (SubVB src1 src2));
+size(4);
+format %{ "VSUB.I8 $dst,$src1,$src2\t! sub packed8B" %}
+ins_encode %{
+bool quad = false;
+__ vsubI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VELEM_SIZE_8, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+instruct vsub16B_reg(vecX dst, vecX src1, vecX src2) %{
+predicate(n->as_Vector()->length() == 16);
+match(Set dst (SubVB src1 src2));
+size(4);
+format %{ "VSUB.I8 $dst.Q,$src1.Q,$src2.Q\t! sub packed16B" %}
+ins_encode %{
+bool quad = true;
+__ vsubI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VELEM_SIZE_8, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// Shorts/Chars vector sub
+instruct vsub4S_reg(vecD dst, vecD src1, vecD src2) %{
+predicate(n->as_Vector()->length() == 4);
+match(Set dst (SubVS src1 src2));
+size(4);
+format %{ "VSUB.I16 $dst,$src1,$src2\t! sub packed4S" %}
+ins_encode %{
+bool quad = false;
+__ vsubI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VELEM_SIZE_16, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+instruct vsub16S_reg(vecX dst, vecX src1, vecX src2) %{
+predicate(n->as_Vector()->length() == 8);
+match(Set dst (SubVS src1 src2));
+size(4);
+format %{ "VSUB.I16 $dst.Q,$src1.Q,$src2.Q\t! sub packed8S" %}
+ins_encode %{
+bool quad = true;
+__ vsubI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VELEM_SIZE_16, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// Integers vector sub
+instruct vsub2I_reg(vecD dst, vecD src1, vecD src2) %{
+predicate(n->as_Vector()->length() == 2);
+match(Set dst (SubVI src1 src2));
+size(4);
+format %{ "VSUB.I32 $dst,$src1,$src2\t! sub packed2I" %}
+ins_encode %{
+bool quad = false;
+__ vsubI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VELEM_SIZE_32, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+instruct vsub4I_reg(vecX dst, vecX src1, vecX src2) %{
+predicate(n->as_Vector()->length() == 4);
+match(Set dst (SubVI src1 src2));
+size(4);
+format %{ "VSUB.I32 $dst.Q,$src1.Q,$src2.Q\t! sub packed4I" %}
+ins_encode %{
+bool quad = true;
+__ vsubI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VELEM_SIZE_32, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// Longs vector sub
+instruct vsub2L_reg(vecX dst, vecX src1, vecX src2) %{
+predicate(n->as_Vector()->length() == 2);
+match(Set dst (SubVL src1 src2));
+size(4);
+format %{ "VSUB.I64 $dst.Q,$src1.Q,$src2.Q\t! sub packed2L" %}
+ins_encode %{
+bool quad = true;
+__ vsubI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VELEM_SIZE_64, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// Floats vector sub
+instruct vsub2F_reg(vecD dst, vecD src1, vecD src2) %{
+predicate(n->as_Vector()->length() == 2 && VM_Version::simd_math_is_compliant());
+match(Set dst (SubVF src1 src2));
+size(4);
+format %{ "VSUB.F32 $dst,$src1,$src2\t! sub packed2F" %}
+ins_encode %{
+bool quad = false;
+__ vsubF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VFA_SIZE_F32, quad);
+%}
+ins_pipe( faddF_reg_reg ); // FIXME
+%}
+#ifndef AARCH64
+instruct vsub2F_reg_vfp(vecD dst, vecD src1, vecD src2) %{
+predicate(n->as_Vector()->length() == 2 && !VM_Version::simd_math_is_compliant());
+match(Set dst (SubVF src1 src2));
+size(4*2);
+ins_cost(DEFAULT_COST*2); // FIXME
+format %{ "FSUBS  $dst.a,$src1.a,$src2.a\n\t"
+"FSUBS  $dst.b,$src1.b,$src2.b" %}
+ins_encode %{
+FloatRegister dsta = $dst$$FloatRegister;
+FloatRegister src1a = $src1$$FloatRegister;
+FloatRegister src2a = $src2$$FloatRegister;
+__ sub_float(dsta, src1a, src2a);
+FloatRegister dstb = dsta->successor();
+FloatRegister src1b = src1a->successor();
+FloatRegister src2b = src2a->successor();
+__ sub_float(dstb, src1b, src2b);
+%}
+ins_pipe(faddF_reg_reg); // FIXME
+%}
+#endif
+instruct vsub4F_reg(vecX dst, vecX src1, vecX src2) %{
+predicate(n->as_Vector()->length() == 4 && VM_Version::simd_math_is_compliant());
+match(Set dst (SubVF src1 src2));
+size(4);
+format %{ "VSUB.F32 $dst.Q,$src1.Q,$src2.Q\t! sub packed4F" %}
+ins_encode %{
+bool quad = true;
+__ vsubF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VFA_SIZE_F32, quad);
+%}
+ins_pipe( faddF_reg_reg ); // FIXME
+%}
+#ifdef AARCH64
+instruct vsub2D_reg_simd(vecX dst, vecX src1, vecX src2) %{
+predicate(n->as_Vector()->length() == 2 && VM_Version::simd_math_is_compliant());
+match(Set dst (SubVD src1 src2));
+size(4);
+format %{ "VSUB.F64 $dst.Q,$src1.Q,$src2.Q\t! add packed2D" %}
+ins_encode %{
+bool quad = true;
+__ vsubF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VFA_SIZE_F64, quad);
+%}
+ins_pipe( faddD_reg_reg ); // FIXME
+%}
+#else
+instruct vsub4F_reg_vfp(vecX dst, vecX src1, vecX src2) %{
+predicate(n->as_Vector()->length() == 4 && !VM_Version::simd_math_is_compliant());
+match(Set dst (SubVF src1 src2));
+size(4*4);
+ins_cost(DEFAULT_COST*4); // FIXME
+format %{ "FSUBS  $dst.a,$src1.a,$src2.a\n\t"
+"FSUBS  $dst.b,$src1.b,$src2.b\n\t"
+"FSUBS  $dst.c,$src1.c,$src2.c\n\t"
+"FSUBS  $dst.d,$src1.d,$src2.d" %}
+ins_encode %{
+FloatRegister dsta = $dst$$FloatRegister;
+FloatRegister src1a = $src1$$FloatRegister;
+FloatRegister src2a = $src2$$FloatRegister;
+__ sub_float(dsta, src1a, src2a);
+FloatRegister dstb = dsta->successor();
+FloatRegister src1b = src1a->successor();
+FloatRegister src2b = src2a->successor();
+__ sub_float(dstb, src1b, src2b);
+FloatRegister dstc = dstb->successor();
+FloatRegister src1c = src1b->successor();
+FloatRegister src2c = src2b->successor();
+__ sub_float(dstc, src1c, src2c);
+FloatRegister dstd = dstc->successor();
+FloatRegister src1d = src1c->successor();
+FloatRegister src2d = src2c->successor();
+__ sub_float(dstd, src1d, src2d);
+%}
+ins_pipe(faddF_reg_reg); // FIXME
+%}
+instruct vsub2D_reg_vfp(vecX dst, vecX src1, vecX src2) %{
+predicate(n->as_Vector()->length() == 2);
+match(Set dst (SubVD src1 src2));
+size(4*2);
+ins_cost(DEFAULT_COST*2); // FIXME
+format %{ "FSUBD  $dst.a,$src1.a,$src2.a\n\t"
+"FSUBD  $dst.b,$src1.b,$src2.b" %}
+ins_encode %{
+FloatRegister dsta = $dst$$FloatRegister;
+FloatRegister src1a = $src1$$FloatRegister;
+FloatRegister src2a = $src2$$FloatRegister;
+__ sub_double(dsta, src1a, src2a);
+FloatRegister dstb = dsta->successor()->successor();
+FloatRegister src1b = src1a->successor()->successor();
+FloatRegister src2b = src2a->successor()->successor();
+__ sub_double(dstb, src1b, src2b);
+%}
+ins_pipe(faddF_reg_reg); // FIXME
+%}
+#endif
+// Shorts/Chars vector mul
+instruct vmul4S_reg(vecD dst, vecD src1, vecD src2) %{
+predicate(n->as_Vector()->length() == 4);
+match(Set dst (MulVS src1 src2));
+size(4);
+format %{ "VMUL.I16 $dst,$src1,$src2\t! mul packed4S" %}
+ins_encode %{
+__ vmulI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VELEM_SIZE_16, 0);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+instruct vmul8S_reg(vecX dst, vecX src1, vecX src2) %{
+predicate(n->as_Vector()->length() == 8);
+match(Set dst (MulVS src1 src2));
+size(4);
+format %{ "VMUL.I16 $dst.Q,$src1.Q,$src2.Q\t! mul packed8S" %}
+ins_encode %{
+__ vmulI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VELEM_SIZE_16, 1);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// Integers vector mul
+instruct vmul2I_reg(vecD dst, vecD src1, vecD src2) %{
+predicate(n->as_Vector()->length() == 2);
+match(Set dst (MulVI src1 src2));
+size(4);
+format %{ "VMUL.I32 $dst,$src1,$src2\t! mul packed2I" %}
+ins_encode %{
+__ vmulI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VELEM_SIZE_32, 0);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+instruct vmul4I_reg(vecX dst, vecX src1, vecX src2) %{
+predicate(n->as_Vector()->length() == 4);
+match(Set dst (MulVI src1 src2));
+size(4);
+format %{ "VMUL.I32 $dst.Q,$src1.Q,$src2.Q\t! mul packed4I" %}
+ins_encode %{
+__ vmulI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VELEM_SIZE_32, 1);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// Floats vector mul
+instruct vmul2F_reg(vecD dst, vecD src1, vecD src2) %{
+predicate(n->as_Vector()->length() == 2 && VM_Version::simd_math_is_compliant());
+match(Set dst (MulVF src1 src2));
+size(4);
+format %{ "VMUL.F32 $dst,$src1,$src2\t! mul packed2F" %}
+ins_encode %{
+__ vmulF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VFA_SIZE_F32, 0);
+%}
+ins_pipe( fmulF_reg_reg ); // FIXME
+%}
+#ifndef AARCH64
+instruct vmul2F_reg_vfp(vecD dst, vecD src1, vecD src2) %{
+predicate(n->as_Vector()->length() == 2 && !VM_Version::simd_math_is_compliant());
+match(Set dst (MulVF src1 src2));
+size(4*2);
+ins_cost(DEFAULT_COST*2); // FIXME
+format %{ "FMULS  $dst.a,$src1.a,$src2.a\n\t"
+"FMULS  $dst.b,$src1.b,$src2.b" %}
+ins_encode %{
+__ mul_float($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
+__ mul_float($dst$$FloatRegister->successor(),
+$src1$$FloatRegister->successor(),
+$src2$$FloatRegister->successor());
+%}
+ins_pipe(fmulF_reg_reg); // FIXME
+%}
+#endif
+instruct vmul4F_reg(vecX dst, vecX src1, vecX src2) %{
+predicate(n->as_Vector()->length() == 4 && VM_Version::simd_math_is_compliant());
+match(Set dst (MulVF src1 src2));
+size(4);
+format %{ "VMUL.F32 $dst.Q,$src1.Q,$src2.Q\t! mul packed4F" %}
+ins_encode %{
+__ vmulF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VFA_SIZE_F32, 1);
+%}
+ins_pipe( fmulF_reg_reg ); // FIXME
+%}
+#ifndef AARCH64
+instruct vmul4F_reg_vfp(vecX dst, vecX src1, vecX src2) %{
+predicate(n->as_Vector()->length() == 4 && !VM_Version::simd_math_is_compliant());
+match(Set dst (MulVF src1 src2));
+size(4*4);
+ins_cost(DEFAULT_COST*4); // FIXME
+format %{ "FMULS  $dst.a,$src1.a,$src2.a\n\t"
+"FMULS  $dst.b,$src1.b,$src2.b\n\t"
+"FMULS  $dst.c,$src1.c,$src2.c\n\t"
+"FMULS  $dst.d,$src1.d,$src2.d" %}
+ins_encode %{
+FloatRegister dsta = $dst$$FloatRegister;
+FloatRegister src1a = $src1$$FloatRegister;
+FloatRegister src2a = $src2$$FloatRegister;
+__ mul_float(dsta, src1a, src2a);
+FloatRegister dstb = dsta->successor();
+FloatRegister src1b = src1a->successor();
+FloatRegister src2b = src2a->successor();
+__ mul_float(dstb, src1b, src2b);
+FloatRegister dstc = dstb->successor();
+FloatRegister src1c = src1b->successor();
+FloatRegister src2c = src2b->successor();
+__ mul_float(dstc, src1c, src2c);
+FloatRegister dstd = dstc->successor();
+FloatRegister src1d = src1c->successor();
+FloatRegister src2d = src2c->successor();
+__ mul_float(dstd, src1d, src2d);
+%}
+ins_pipe(fmulF_reg_reg); // FIXME
+%}
+#endif
+#ifdef AARCH64
+instruct vmul2D_reg(vecX dst, vecX src1, vecX src2) %{
+predicate(n->as_Vector()->length() == 2 && VM_Version::has_simd());
+match(Set dst (MulVD src1 src2));
+size(4*1);
+ins_cost(DEFAULT_COST*1); // FIXME
+format %{ "FMUL.2D $dst,$src1,$src2\t! double[2]" %}
+ins_encode %{
+int quad = 1;
+__ vmulF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VFA_SIZE_F64, quad);
+%}
+ins_pipe(fdivF_reg_reg); // FIXME
+%}
+#else
+instruct vmul2D_reg_vfp(vecX dst, vecX src1, vecX src2) %{
+predicate(n->as_Vector()->length() == 2);
+match(Set dst (MulVD src1 src2));
+size(4*2);
+ins_cost(DEFAULT_COST*2); // FIXME
+format %{ "FMULD  $dst.D.a,$src1.D.a,$src2.D.a\n\t"
+"FMULD  $dst.D.b,$src1.D.b,$src2.D.b" %}
+ins_encode %{
+FloatRegister dsta = $dst$$FloatRegister;
+FloatRegister src1a = $src1$$FloatRegister;
+FloatRegister src2a = $src2$$FloatRegister;
+__ mul_double(dsta, src1a, src2a);
+FloatRegister dstb = dsta->successor()->successor();
+FloatRegister src1b = src1a->successor()->successor();
+FloatRegister src2b = src2a->successor()->successor();
+__ mul_double(dstb, src1b, src2b);
+%}
+ins_pipe(fmulD_reg_reg); // FIXME
+%}
+#endif
+// Floats vector div
+instruct vdiv2F_reg_vfp(vecD dst, vecD src1, vecD src2) %{
+predicate(n->as_Vector()->length() == 2);
+match(Set dst (DivVF src1 src2));
+#ifdef AARCH64
+size(4*1);
+ins_cost(DEFAULT_COST*1); // FIXME
+format %{ "FDIV.2S $dst,$src1,$src2\t! float[2]" %}
+ins_encode %{
+int quad = 0;
+__ vdivF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VFA_SIZE_F32, quad);
+%}
+ins_pipe(fdivF_reg_reg); // FIXME
+#else
+size(4*2);
+ins_cost(DEFAULT_COST*2); // FIXME
+format %{ "FDIVS  $dst.a,$src1.a,$src2.a\n\t"
+"FDIVS  $dst.b,$src1.b,$src2.b" %}
+ins_encode %{
+__ div_float($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
+__ div_float($dst$$FloatRegister->successor(),
+$src1$$FloatRegister->successor(),
+$src2$$FloatRegister->successor());
+%}
+ins_pipe(fdivF_reg_reg); // FIXME
+#endif
+%}
+instruct vdiv4F_reg_vfp(vecX dst, vecX src1, vecX src2) %{
+predicate(n->as_Vector()->length() == 4);
+match(Set dst (DivVF src1 src2));
+#ifdef AARCH64
+size(4*1);
+ins_cost(DEFAULT_COST*1); // FIXME
+format %{ "FDIV.4S $dst,$src1,$src2\t! float[4]" %}
+ins_encode %{
+int quad = 1;
+__ vdivF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VFA_SIZE_F32, quad);
+%}
+ins_pipe(fdivF_reg_reg); // FIXME
+#else
+size(4*4);
+ins_cost(DEFAULT_COST*4); // FIXME
+format %{ "FDIVS  $dst.a,$src1.a,$src2.a\n\t"
+"FDIVS  $dst.b,$src1.b,$src2.b\n\t"
+"FDIVS  $dst.c,$src1.c,$src2.c\n\t"
+"FDIVS  $dst.d,$src1.d,$src2.d" %}
+ins_encode %{
+FloatRegister dsta = $dst$$FloatRegister;
+FloatRegister src1a = $src1$$FloatRegister;
+FloatRegister src2a = $src2$$FloatRegister;
+__ div_float(dsta, src1a, src2a);
+FloatRegister dstb = dsta->successor();
+FloatRegister src1b = src1a->successor();
+FloatRegister src2b = src2a->successor();
+__ div_float(dstb, src1b, src2b);
+FloatRegister dstc = dstb->successor();
+FloatRegister src1c = src1b->successor();
+FloatRegister src2c = src2b->successor();
+__ div_float(dstc, src1c, src2c);
+FloatRegister dstd = dstc->successor();
+FloatRegister src1d = src1c->successor();
+FloatRegister src2d = src2c->successor();
+__ div_float(dstd, src1d, src2d);
+%}
+ins_pipe(fdivF_reg_reg); // FIXME
+#endif
+%}
+#ifdef AARCH64
+instruct vdiv2D_reg(vecX dst, vecX src1, vecX src2) %{
+predicate(n->as_Vector()->length() == 2 && VM_Version::has_simd());
+match(Set dst (DivVD src1 src2));
+size(4*1);
+ins_cost(DEFAULT_COST*1); // FIXME
+format %{ "FDIV.2D $dst,$src1,$src2\t! double[2]" %}
+ins_encode %{
+int quad = 1;
+__ vdivF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+MacroAssembler::VFA_SIZE_F64, quad);
+%}
+ins_pipe(fdivF_reg_reg); // FIXME
+%}
+#else
+instruct vdiv2D_reg_vfp(vecX dst, vecX src1, vecX src2) %{
+predicate(n->as_Vector()->length() == 2);
+match(Set dst (DivVD src1 src2));
+size(4*2);
+ins_cost(DEFAULT_COST*2); // FIXME
+format %{ "FDIVD  $dst.D.a,$src1.D.a,$src2.D.a\n\t"
+"FDIVD  $dst.D.b,$src1.D.b,$src2.D.b" %}
+ins_encode %{
+FloatRegister dsta = $dst$$FloatRegister;
+FloatRegister src1a = $src1$$FloatRegister;
+FloatRegister src2a = $src2$$FloatRegister;
+__ div_double(dsta, src1a, src2a);
+FloatRegister dstb = dsta->successor()->successor();
+FloatRegister src1b = src1a->successor()->successor();
+FloatRegister src2b = src2a->successor()->successor();
+__ div_double(dstb, src1b, src2b);
+%}
+ins_pipe(fdivD_reg_reg); // FIXME
+%}
+#endif
+// --------------------------------- NEG --------------------------------------
+instruct vneg8B_reg(vecD dst, vecD src) %{
+predicate(n->as_Vector()->length_in_bytes() == 8);
+effect(DEF dst, USE src);
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{ "VNEG.S8 $dst.D,$src.D\t! neg packed8B" %}
+ins_encode %{
+bool quad = false;
+__ vnegI($dst$$FloatRegister, $src$$FloatRegister,
+MacroAssembler::VELEM_SIZE_8, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+instruct vneg16B_reg(vecX dst, vecX src) %{
+predicate(n->as_Vector()->length_in_bytes() == 16);
+effect(DEF dst, USE src);
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{ "VNEG.S8 $dst.Q,$src.Q\t! neg0 packed16B" %}
+ins_encode %{
+bool _float = false;
+bool quad = true;
+__ vnegI($dst$$FloatRegister, $src$$FloatRegister,
+MacroAssembler::VELEM_SIZE_8, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// ------------------------------ Shift ---------------------------------------
+instruct vslcntD(vecD dst, iRegI cnt) %{
+predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
+match(Set dst (LShiftCntV cnt));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+expand %{
+Repl8B_reg_simd(dst, cnt);
+%}
+%}
+instruct vslcntX(vecX dst, iRegI cnt) %{
+predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
+match(Set dst (LShiftCntV cnt));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+expand %{
+Repl16B_reg(dst, cnt);
+%}
+%}
+// Low bits of vector "shift" elements are used, so it
+// doesn't matter if we treat it as ints or bytes here.
+instruct vsrcntD(vecD dst, iRegI cnt) %{
+predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
+match(Set dst (RShiftCntV cnt));
+size(4*2);
+ins_cost(DEFAULT_COST*2); // FIXME
+format %{ "VDUP.8 $dst.D,$cnt\n\t"
+"VNEG.S8 $dst.D,$dst.D\t! neg packed8B" %}
+ins_encode %{
+bool quad = false;
+__ vdupI($dst$$FloatRegister, $cnt$$Register,
+MacroAssembler::VELEM_SIZE_8, quad);
+__ vnegI($dst$$FloatRegister, $dst$$FloatRegister,
+MacroAssembler::VELEM_SIZE_8, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+instruct vsrcntX(vecX dst, iRegI cnt) %{
+predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
+match(Set dst (RShiftCntV cnt));
+size(4*2);
+ins_cost(DEFAULT_COST*2); // FIXME
+format %{ "VDUP.8 $dst.Q,$cnt\n\t"
+"VNEG.S8 $dst.Q,$dst.Q\t! neg packed16B" %}
+ins_encode %{
+bool quad = true;
+__ vdupI($dst$$FloatRegister, $cnt$$Register,
+MacroAssembler::VELEM_SIZE_8, quad);
+__ vnegI($dst$$FloatRegister, $dst$$FloatRegister,
+MacroAssembler::VELEM_SIZE_8, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// Byte vector logical left/right shift based on sign
+instruct vsh8B_reg(vecD dst, vecD src, vecD shift) %{
+predicate(n->as_Vector()->length() == 8);
+effect(DEF dst, USE src, USE shift);
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHL.U8 $dst.D,$src.D,$shift.D\t! logical left/right shift packed8B"
+%}
+ins_encode %{
+bool quad = false;
+__ vshlUI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
+MacroAssembler::VELEM_SIZE_8, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+instruct vsh16B_reg(vecX dst, vecX src, vecX shift) %{
+predicate(n->as_Vector()->length() == 16);
+effect(DEF dst, USE src, USE shift);
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHL.U8 $dst.Q,$src.Q,$shift.Q\t! logical left/right shift packed16B"
+%}
+ins_encode %{
+bool quad = true;
+__ vshlUI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
+MacroAssembler::VELEM_SIZE_8, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// Shorts/Char vector logical left/right shift based on sign
+instruct vsh4S_reg(vecD dst, vecD src, vecD shift) %{
+predicate(n->as_Vector()->length() == 4);
+effect(DEF dst, USE src, USE shift);
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHL.U16 $dst.D,$src.D,$shift.D\t! logical left/right shift packed4S"
+%}
+ins_encode %{
+bool quad = false;
+__ vshlUI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
+MacroAssembler::VELEM_SIZE_16, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+instruct vsh8S_reg(vecX dst, vecX src, vecX shift) %{
+predicate(n->as_Vector()->length() == 8);
+effect(DEF dst, USE src, USE shift);
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHL.U16 $dst.Q,$src.Q,$shift.Q\t! logical left/right shift packed8S"
+%}
+ins_encode %{
+bool quad = true;
+__ vshlUI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
+MacroAssembler::VELEM_SIZE_16, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// Integers vector logical left/right shift based on sign
+instruct vsh2I_reg(vecD dst, vecD src, vecD shift) %{
+predicate(n->as_Vector()->length() == 2);
+effect(DEF dst, USE src, USE shift);
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHL.U32 $dst.D,$src.D,$shift.D\t! logical left/right shift packed2I"
+%}
+ins_encode %{
+bool quad = false;
+__ vshlUI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
+MacroAssembler::VELEM_SIZE_32, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+instruct vsh4I_reg(vecX dst, vecX src, vecX shift) %{
+predicate(n->as_Vector()->length() == 4);
+effect(DEF dst, USE src, USE shift);
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHL.U32 $dst.Q,$src.Q,$shift.Q\t! logical left/right shift packed4I"
+%}
+ins_encode %{
+bool quad = true;
+__ vshlUI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
+MacroAssembler::VELEM_SIZE_32, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// Longs vector logical left/right shift based on sign
+instruct vsh2L_reg(vecX dst, vecX src, vecX shift) %{
+predicate(n->as_Vector()->length() == 2);
+effect(DEF dst, USE src, USE shift);
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHL.U64 $dst.Q,$src.Q,$shift.Q\t! logical left/right shift packed2L"
+%}
+ins_encode %{
+bool quad = true;
+__ vshlUI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
+MacroAssembler::VELEM_SIZE_64, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// ------------------------------ LeftShift -----------------------------------
+// Byte vector left shift
+instruct vsl8B_reg(vecD dst, vecD src, vecD shift) %{
+predicate(n->as_Vector()->length() == 8);
+match(Set dst (LShiftVB src shift));
+size(4*1);
+ins_cost(DEFAULT_COST*1); // FIXME
+expand %{
+vsh8B_reg(dst, src, shift);
+%}
+%}
+instruct vsl16B_reg(vecX dst, vecX src, vecX shift) %{
+predicate(n->as_Vector()->length() == 16);
+match(Set dst (LShiftVB src shift));
+size(4*1);
+ins_cost(DEFAULT_COST*1); // FIXME
+expand %{
+vsh16B_reg(dst, src, shift);
+%}
+%}
+instruct vsl8B_immI(vecD dst, vecD src, immI shift) %{
+predicate(n->as_Vector()->length() == 8);
+match(Set dst (LShiftVB src shift));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHL.I8 $dst.D,$src.D,$shift\t! logical left shift packed8B"
+%}
+ins_encode %{
+bool quad = false;
+__ vshli($dst$$FloatRegister, $src$$FloatRegister, 8, $shift$$constant,
+quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+instruct vsl16B_immI(vecX dst, vecX src, immI shift) %{
+predicate(n->as_Vector()->length() == 16);
+match(Set dst (LShiftVB src shift));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHL.I8 $dst.Q,$src.Q,$shift\t! logical left shift packed16B"
+%}
+ins_encode %{
+bool quad = true;
+__ vshli($dst$$FloatRegister, $src$$FloatRegister, 8, $shift$$constant,
+quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// Shorts/Chars vector logical left/right shift
+instruct vsl4S_reg(vecD dst, vecD src, vecD shift) %{
+predicate(n->as_Vector()->length() == 4);
+match(Set dst (LShiftVS src shift));
+match(Set dst (URShiftVS src shift));
+size(4*1);
+ins_cost(DEFAULT_COST*1); // FIXME
+expand %{
+vsh4S_reg(dst, src, shift);
+%}
+%}
+instruct vsl8S_reg(vecX dst, vecX src, vecX shift) %{
+predicate(n->as_Vector()->length() == 8);
+match(Set dst (LShiftVS src shift));
+match(Set dst (URShiftVS src shift));
+size(4*1);
+ins_cost(DEFAULT_COST*1); // FIXME
+expand %{
+vsh8S_reg(dst, src, shift);
+%}
+%}
+instruct vsl4S_immI(vecD dst, vecD src, immI shift) %{
+predicate(n->as_Vector()->length() == 4);
+match(Set dst (LShiftVS src shift));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHL.I16 $dst.D,$src.D,$shift\t! logical left shift packed4S"
+%}
+ins_encode %{
+bool quad = false;
+__ vshli($dst$$FloatRegister, $src$$FloatRegister, 16, $shift$$constant,
+quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+instruct vsl8S_immI(vecX dst, vecX src, immI shift) %{
+predicate(n->as_Vector()->length() == 8);
+match(Set dst (LShiftVS src shift));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHL.I16 $dst.Q,$src.Q,$shift\t! logical left shift packed8S"
+%}
+ins_encode %{
+bool quad = true;
+__ vshli($dst$$FloatRegister, $src$$FloatRegister, 16, $shift$$constant,
+quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// Integers vector logical left/right shift
+instruct vsl2I_reg(vecD dst, vecD src, vecD shift) %{
+predicate(n->as_Vector()->length() == 2 && VM_Version::has_simd());
+match(Set dst (LShiftVI src shift));
+match(Set dst (URShiftVI src shift));
+size(4*1);
+ins_cost(DEFAULT_COST*1); // FIXME
+expand %{
+vsh2I_reg(dst, src, shift);
+%}
+%}
+instruct vsl4I_reg(vecX dst, vecX src, vecX shift) %{
+predicate(n->as_Vector()->length() == 4 && VM_Version::has_simd());
+match(Set dst (LShiftVI src shift));
+match(Set dst (URShiftVI src shift));
+size(4*1);
+ins_cost(DEFAULT_COST*1); // FIXME
+expand %{
+vsh4I_reg(dst, src, shift);
+%}
+%}
+instruct vsl2I_immI(vecD dst, vecD src, immI shift) %{
+predicate(n->as_Vector()->length() == 2 && VM_Version::has_simd());
+match(Set dst (LShiftVI src shift));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHL.I32 $dst.D,$src.D,$shift\t! logical left shift packed2I"
+%}
+ins_encode %{
+bool quad = false;
+__ vshli($dst$$FloatRegister, $src$$FloatRegister, 32, $shift$$constant,
+quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+instruct vsl4I_immI(vecX dst, vecX src, immI shift) %{
+predicate(n->as_Vector()->length() == 4 && VM_Version::has_simd());
+match(Set dst (LShiftVI src shift));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHL.I32 $dst.Q,$src.Q,$shift\t! logical left shift packed4I"
+%}
+ins_encode %{
+bool quad = true;
+__ vshli($dst$$FloatRegister, $src$$FloatRegister, 32, $shift$$constant,
+quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// Longs vector logical left/right shift
+instruct vsl2L_reg(vecX dst, vecX src, vecX shift) %{
+predicate(n->as_Vector()->length() == 2);
+match(Set dst (LShiftVL src shift));
+match(Set dst (URShiftVL src shift));
+size(4*1);
+ins_cost(DEFAULT_COST*1); // FIXME
+expand %{
+vsh2L_reg(dst, src, shift);
+%}
+%}
+instruct vsl2L_immI(vecX dst, vecX src, immI shift) %{
+predicate(n->as_Vector()->length() == 2);
+match(Set dst (LShiftVL src shift));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHL.I64 $dst.Q,$src.Q,$shift\t! logical left shift packed2L"
+%}
+ins_encode %{
+bool quad = true;
+__ vshli($dst$$FloatRegister, $src$$FloatRegister, 64, $shift$$constant,
+quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// ----------------------- LogicalRightShift -----------------------------------
+// Bytes/Shorts vector logical right shift produces incorrect Java result
+// for negative data because java code convert short value into int with
+// sign extension before a shift.
+// Chars vector logical right shift
+instruct vsrl4S_immI(vecD dst, vecD src, immI shift) %{
+predicate(n->as_Vector()->length() == 4);
+match(Set dst (URShiftVS src shift));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHR.U16 $dst.D,$src.D,$shift\t! logical right shift packed4S"
+%}
+ins_encode %{
+bool quad = false;
+__ vshrUI($dst$$FloatRegister, $src$$FloatRegister, 16, $shift$$constant,
+quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+instruct vsrl8S_immI(vecX dst, vecX src, immI shift) %{
+predicate(n->as_Vector()->length() == 8);
+match(Set dst (URShiftVS src shift));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHR.U16 $dst.Q,$src.Q,$shift\t! logical right shift packed8S"
+%}
+ins_encode %{
+bool quad = true;
+__ vshrUI($dst$$FloatRegister, $src$$FloatRegister, 16, $shift$$constant,
+quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// Integers vector logical right shift
+instruct vsrl2I_immI(vecD dst, vecD src, immI shift) %{
+predicate(n->as_Vector()->length() == 2 && VM_Version::has_simd());
+match(Set dst (URShiftVI src shift));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHR.U32 $dst.D,$src.D,$shift\t! logical right shift packed2I"
+%}
+ins_encode %{
+bool quad = false;
+__ vshrUI($dst$$FloatRegister, $src$$FloatRegister, 32, $shift$$constant,
+quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+instruct vsrl4I_immI(vecX dst, vecX src, immI shift) %{
+predicate(n->as_Vector()->length() == 4 && VM_Version::has_simd());
+match(Set dst (URShiftVI src shift));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHR.U32 $dst.Q,$src.Q,$shift\t! logical right shift packed4I"
+%}
+ins_encode %{
+bool quad = true;
+__ vshrUI($dst$$FloatRegister, $src$$FloatRegister, 32, $shift$$constant,
+quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// Longs vector logical right shift
+instruct vsrl2L_immI(vecX dst, vecX src, immI shift) %{
+predicate(n->as_Vector()->length() == 2);
+match(Set dst (URShiftVL src shift));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHR.U64 $dst.Q,$src.Q,$shift\t! logical right shift packed2L"
+%}
+ins_encode %{
+bool quad = true;
+__ vshrUI($dst$$FloatRegister, $src$$FloatRegister, 64, $shift$$constant,
+quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// ------------------- ArithmeticRightShift -----------------------------------
+// Bytes vector arithmetic left/right shift based on sign
+instruct vsha8B_reg(vecD dst, vecD src, vecD shift) %{
+predicate(n->as_Vector()->length() == 8);
+effect(DEF dst, USE src, USE shift);
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHL.S8 $dst.D,$src.D,$shift.D\t! arithmetic right shift packed8B"
+%}
+ins_encode %{
+bool quad = false;
+__ vshlSI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
+MacroAssembler::VELEM_SIZE_8, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+instruct vsha16B_reg(vecX dst, vecX src, vecX shift) %{
+predicate(n->as_Vector()->length() == 16);
+effect(DEF dst, USE src, USE shift);
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHL.S8 $dst.Q,$src.Q,$shift.Q\t! arithmetic right shift packed16B"
+%}
+ins_encode %{
+bool quad = true;
+__ vshlSI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
+MacroAssembler::VELEM_SIZE_8, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// Shorts vector arithmetic left/right shift based on sign
+instruct vsha4S_reg(vecD dst, vecD src, vecD shift) %{
+predicate(n->as_Vector()->length() == 4);
+effect(DEF dst, USE src, USE shift);
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHL.S16 $dst.D,$src.D,$shift.D\t! arithmetic right shift packed4S"
+%}
+ins_encode %{
+bool quad = false;
+__ vshlSI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
+MacroAssembler::VELEM_SIZE_16, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+instruct vsha8S_reg(vecX dst, vecX src, vecX shift) %{
+predicate(n->as_Vector()->length() == 8);
+effect(DEF dst, USE src, USE shift);
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHL.S16 $dst.Q,$src.Q,$shift.Q\t! arithmetic right shift packed8S"
+%}
+ins_encode %{
+bool quad = true;
+__ vshlSI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
+MacroAssembler::VELEM_SIZE_16, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// Integers vector arithmetic left/right shift based on sign
+instruct vsha2I_reg(vecD dst, vecD src, vecD shift) %{
+predicate(n->as_Vector()->length() == 2);
+effect(DEF dst, USE src, USE shift);
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHL.S32 $dst.D,$src.D,$shift.D\t! arithmetic right shift packed2I"
+%}
+ins_encode %{
+bool quad = false;
+__ vshlSI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
+MacroAssembler::VELEM_SIZE_32, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+instruct vsha4I_reg(vecX dst, vecX src, vecX shift) %{
+predicate(n->as_Vector()->length() == 4);
+effect(DEF dst, USE src, USE shift);
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHL.S32 $dst.Q,$src.Q,$shift.Q\t! arithmetic right shift packed4I"
+%}
+ins_encode %{
+bool quad = true;
+__ vshlSI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
+MacroAssembler::VELEM_SIZE_32, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// Longs vector arithmetic left/right shift based on sign
+instruct vsha2L_reg(vecX dst, vecX src, vecX shift) %{
+predicate(n->as_Vector()->length() == 2);
+effect(DEF dst, USE src, USE shift);
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHL.S64 $dst.Q,$src.Q,$shift.Q\t! arithmetic right shift packed2L"
+%}
+ins_encode %{
+bool quad = true;
+__ vshlSI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
+MacroAssembler::VELEM_SIZE_64, quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// Byte vector arithmetic right shift
+instruct vsra8B_reg(vecD dst, vecD src, vecD shift) %{
+predicate(n->as_Vector()->length() == 8);
+match(Set dst (RShiftVB src shift));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+expand %{
+vsha8B_reg(dst, src, shift);
+%}
+%}
+instruct vsrl16B_reg(vecX dst, vecX src, vecX shift) %{
+predicate(n->as_Vector()->length() == 16);
+match(Set dst (RShiftVB src shift));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+expand %{
+vsha16B_reg(dst, src, shift);
+%}
+%}
+instruct vsrl8B_immI(vecD dst, vecD src, immI shift) %{
+predicate(n->as_Vector()->length() == 8);
+match(Set dst (RShiftVB src shift));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHR.S8 $dst.D,$src.D,$shift\t! logical right shift packed8B"
+%}
+ins_encode %{
+bool quad = false;
+__ vshrSI($dst$$FloatRegister, $src$$FloatRegister, 8, $shift$$constant,
+quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+instruct vsrl16B_immI(vecX dst, vecX src, immI shift) %{
+predicate(n->as_Vector()->length() == 16);
+match(Set dst (RShiftVB src shift));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHR.S8 $dst.Q,$src.Q,$shift\t! logical right shift packed16B"
+%}
+ins_encode %{
+bool quad = true;
+__ vshrSI($dst$$FloatRegister, $src$$FloatRegister, 8, $shift$$constant,
+quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// Shorts vector arithmetic right shift
+instruct vsra4S_reg(vecD dst, vecD src, vecD shift) %{
+predicate(n->as_Vector()->length() == 4);
+match(Set dst (RShiftVS src shift));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+expand %{
+vsha4S_reg(dst, src, shift);
+%}
+%}
+instruct vsra8S_reg(vecX dst, vecX src, vecX shift) %{
+predicate(n->as_Vector()->length() == 8);
+match(Set dst (RShiftVS src shift));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+expand %{
+vsha8S_reg(dst, src, shift);
+%}
+%}
+instruct vsra4S_immI(vecD dst, vecD src, immI shift) %{
+predicate(n->as_Vector()->length() == 4);
+match(Set dst (RShiftVS src shift));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHR.S16 $dst.D,$src.D,$shift\t! logical right shift packed4S"
+%}
+ins_encode %{
+bool quad = false;
+__ vshrSI($dst$$FloatRegister, $src$$FloatRegister, 16, $shift$$constant,
+quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+instruct vsra8S_immI(vecX dst, vecX src, immI shift) %{
+predicate(n->as_Vector()->length() == 8);
+match(Set dst (RShiftVS src shift));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHR.S16 $dst.Q,$src.Q,$shift\t! logical right shift packed8S"
+%}
+ins_encode %{
+bool quad = true;
+__ vshrSI($dst$$FloatRegister, $src$$FloatRegister, 16, $shift$$constant,
+quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// Integers vector arithmetic right shift
+instruct vsra2I_reg(vecD dst, vecD src, vecD shift) %{
+predicate(n->as_Vector()->length() == 2);
+match(Set dst (RShiftVI src shift));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+expand %{
+vsha2I_reg(dst, src, shift);
+%}
+%}
+instruct vsra4I_reg(vecX dst, vecX src, vecX shift) %{
+predicate(n->as_Vector()->length() == 4);
+match(Set dst (RShiftVI src shift));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+expand %{
+vsha4I_reg(dst, src, shift);
+%}
+%}
+instruct vsra2I_immI(vecD dst, vecD src, immI shift) %{
+predicate(n->as_Vector()->length() == 2);
+match(Set dst (RShiftVI src shift));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHR.S32 $dst.D,$src.D,$shift\t! logical right shift packed2I"
+%}
+ins_encode %{
+bool quad = false;
+__ vshrSI($dst$$FloatRegister, $src$$FloatRegister, 32, $shift$$constant,
+quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+instruct vsra4I_immI(vecX dst, vecX src, immI shift) %{
+predicate(n->as_Vector()->length() == 4);
+match(Set dst (RShiftVI src shift));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHR.S32 $dst.Q,$src.Q,$shift\t! logical right shift packed4I"
+%}
+ins_encode %{
+bool quad = true;
+__ vshrSI($dst$$FloatRegister, $src$$FloatRegister, 32, $shift$$constant,
+quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// Longs vector arithmetic right shift
+instruct vsra2L_reg(vecX dst, vecX src, vecX shift) %{
+predicate(n->as_Vector()->length() == 2);
+match(Set dst (RShiftVL src shift));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+expand %{
+vsha2L_reg(dst, src, shift);
+%}
+%}
+instruct vsra2L_immI(vecX dst, vecX src, immI shift) %{
+predicate(n->as_Vector()->length() == 2);
+match(Set dst (RShiftVL src shift));
+size(4);
+ins_cost(DEFAULT_COST); // FIXME
+format %{
+"VSHR.S64 $dst.Q,$src.Q,$shift\t! logical right shift packed2L"
+%}
+ins_encode %{
+bool quad = true;
+__ vshrSI($dst$$FloatRegister, $src$$FloatRegister, 64, $shift$$constant,
+quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// --------------------------------- AND --------------------------------------
+instruct vandD(vecD dst, vecD src1, vecD src2) %{
+predicate(n->as_Vector()->length_in_bytes() == 8);
+match(Set dst (AndV src1 src2));
+format %{ "VAND    $dst.D,$src1.D,$src2.D\t! and vectors (8 bytes)" %}
+ins_encode %{
+bool quad = false;
+__ vandI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+instruct vandX(vecX dst, vecX src1, vecX src2) %{
+predicate(n->as_Vector()->length_in_bytes() == 16);
+match(Set dst (AndV src1 src2));
+format %{ "VAND    $dst.Q,$src1.Q,$src2.Q\t! and vectors (16 bytes)" %}
+ins_encode %{
+bool quad = true;
+__ vandI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// --------------------------------- OR ---------------------------------------
+instruct vorD(vecD dst, vecD src1, vecD src2) %{
+predicate(n->as_Vector()->length_in_bytes() == 8);
+match(Set dst (OrV src1 src2));
+format %{ "VOR     $dst.D,$src1.D,$src2.D\t! and vectors (8 bytes)" %}
+ins_encode %{
+bool quad = false;
+__ vorI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+instruct vorX(vecX dst, vecX src1, vecX src2) %{
+predicate(n->as_Vector()->length_in_bytes() == 16);
+match(Set dst (OrV src1 src2));
+format %{ "VOR     $dst.Q,$src1.Q,$src2.Q\t! and vectors (16 bytes)" %}
+ins_encode %{
+bool quad = true;
+__ vorI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+// --------------------------------- XOR --------------------------------------
+instruct vxorD(vecD dst, vecD src1, vecD src2) %{
+predicate(n->as_Vector()->length_in_bytes() == 8);
+match(Set dst (XorV src1 src2));
+format %{ "VXOR    $dst.D,$src1.D,$src2.D\t! and vectors (8 bytes)" %}
+ins_encode %{
+bool quad = false;
+__ vxorI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+instruct vxorX(vecX dst, vecX src1, vecX src2) %{
+predicate(n->as_Vector()->length_in_bytes() == 16);
+match(Set dst (XorV src1 src2));
+format %{ "VXOR    $dst.Q,$src1.Q,$src2.Q\t! and vectors (16 bytes)" %}
+ins_encode %{
+bool quad = true;
+__ vxorI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
+quad);
+%}
+ins_pipe( ialu_reg_reg ); // FIXME
+%}
+//----------PEEPHOLE RULES-----------------------------------------------------
+// These must follow all instruction definitions as they use the names
+// defined in the instructions definitions.
+//
+// peepmatch ( root_instr_name [preceding_instruction]* );
+//
+// peepconstraint %{
+// (instruction_number.operand_name relational_op instruction_number.operand_name
+//  [, ...] );
+// // instruction numbers are zero-based using left to right order in peepmatch
+//
+// peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
+// // provide an instruction_number.operand_name for each operand that appears
+// // in the replacement instruction's match rule
+//
+// ---------VM FLAGS---------------------------------------------------------
+//
+// All peephole optimizations can be turned off using -XX:-OptoPeephole
+//
+// Each peephole rule is given an identifying number starting with zero and
+// increasing by one in the order seen by the parser.  An individual peephole
+// can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
+// on the command-line.
+//
+// ---------CURRENT LIMITATIONS----------------------------------------------
+//
+// Only match adjacent instructions in same basic block
+// Only equality constraints
+// Only constraints between operands, not (0.dest_reg == EAX_enc)
+// Only one replacement instruction
+//
+// ---------EXAMPLE----------------------------------------------------------
+//
+// // pertinent parts of existing instructions in architecture description
+// instruct movI(eRegI dst, eRegI src) %{
+//   match(Set dst (CopyI src));
+// %}
+//
+// instruct incI_eReg(eRegI dst, immI1 src, eFlagsReg cr) %{
+//   match(Set dst (AddI dst src));
+//   effect(KILL cr);
+// %}
+//
+// // Change (inc mov) to lea
+// peephole %{
+//   // increment preceeded by register-register move
+//   peepmatch ( incI_eReg movI );
+//   // require that the destination register of the increment
+//   // match the destination register of the move
+//   peepconstraint ( 0.dst == 1.dst );
+//   // construct a replacement instruction that sets
+//   // the destination to ( move's source register + one )
+//   peepreplace ( incI_eReg_immI1( 0.dst 1.src 0.src ) );
+// %}
+//
+// // Change load of spilled value to only a spill
+// instruct storeI(memory mem, eRegI src) %{
+//   match(Set mem (StoreI mem src));
+// %}
+//
+// instruct loadI(eRegI dst, memory mem) %{
+//   match(Set dst (LoadI mem));
+// %}
+//
+// peephole %{
+//   peepmatch ( loadI storeI );
+//   peepconstraint ( 1.src == 0.dst, 1.mem == 0.mem );
+//   peepreplace ( storeI( 1.mem 1.mem 1.src ) );
+// %}
+//----------SMARTSPILL RULES---------------------------------------------------
+// These must follow all instruction definitions as they use the names
+// defined in the instructions definitions.
+//
+// ARM will probably not have any of these rules due to RISC instruction set.
+//----------PIPELINE-----------------------------------------------------------
+// Rules which define the behavior of the target architectures pipeline.

changeset 42664	29142a56c193
child 45965	e29c1363af9a
child 46378	4ccca1fdf627