/*

 * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved.

 * Copyright (c) 2014, 2019, Red Hat Inc. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#ifndef CPU_AARCH64_MACROASSEMBLER_AARCH64_HPP

#define CPU_AARCH64_MACROASSEMBLER_AARCH64_HPP

#include "asm/assembler.hpp"

#include "oops/compressedOops.hpp"

// MacroAssembler extends Assembler by frequently used macros.

//

// Instructions for which a 'better' code sequence exists depending

// on arguments should also go in here.

class MacroAssembler: public Assembler {

  friend class LIR_Assembler;

 public:

  using Assembler::mov;

  using Assembler::movi;

 protected:

  // Support for VM calls

  //

  // This is the base routine called by the different versions of call_VM_leaf. The interpreter

  // may customize this version by overriding it for its purposes (e.g., to save/restore

  // additional registers when doing a VM call).

  virtual void call_VM_leaf_base(

    address entry_point,               // the entry point

    int     number_of_arguments,        // the number of arguments to pop after the call

    Label *retaddr = NULL

  );

  virtual void call_VM_leaf_base(

    address entry_point,               // the entry point

    int     number_of_arguments,        // the number of arguments to pop after the call

    Label &retaddr) {

    call_VM_leaf_base(entry_point, number_of_arguments, &retaddr);

  }

  // This is the base routine called by the different versions of call_VM. The interpreter

  // may customize this version by overriding it for its purposes (e.g., to save/restore

  // additional registers when doing a VM call).

  //

  // If no java_thread register is specified (noreg) than rthread will be used instead. call_VM_base

  // returns the register which contains the thread upon return. If a thread register has been

  // specified, the return value will correspond to that register. If no last_java_sp is specified

  // (noreg) than rsp will be used instead.

  virtual void call_VM_base(           // returns the register containing the thread upon return

    Register oop_result,               // where an oop-result ends up if any; use noreg otherwise

    Register java_thread,              // the thread if computed before     ; use noreg otherwise

    Register last_java_sp,             // to set up last_Java_frame in stubs; use noreg otherwise

    address  entry_point,              // the entry point

    int      number_of_arguments,      // the number of arguments (w/o thread) to pop after the call

    bool     check_exceptions          // whether to check for pending exceptions after return

  );

  void call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions = true);

  // True if an XOR can be used to expand narrow klass references.

  bool use_XOR_for_compressed_class_base;

 public:

  MacroAssembler(CodeBuffer* code) : Assembler(code) {

    use_XOR_for_compressed_class_base

      = operand_valid_for_logical_immediate

           (/*is32*/false, (uint64_t)CompressedKlassPointers::base())

         && ((uint64_t)CompressedKlassPointers::base()

             > (1UL << log2_intptr(CompressedKlassPointers::range())));

  }

 // These routines should emit JVMTI PopFrame and ForceEarlyReturn handling code.

 // The implementation is only non-empty for the InterpreterMacroAssembler,

 // as only the interpreter handles PopFrame and ForceEarlyReturn requests.

 virtual void check_and_handle_popframe(Register java_thread);

 virtual void check_and_handle_earlyret(Register java_thread);

  void safepoint_poll(Label& slow_path);

  void safepoint_poll_acquire(Label& slow_path);

  // Biased locking support

  // lock_reg and obj_reg must be loaded up with the appropriate values.

  // swap_reg is killed.

  // tmp_reg must be supplied and must not be rscratch1 or rscratch2

  // Optional slow case is for implementations (interpreter and C1) which branch to

  // slow case directly. Leaves condition codes set for C2's Fast_Lock node.

  // Returns offset of first potentially-faulting instruction for null

  // check info (currently consumed only by C1). If

  // swap_reg_contains_mark is true then returns -1 as it is assumed

  // the calling code has already passed any potential faults.

  int biased_locking_enter(Register lock_reg, Register obj_reg,

                           Register swap_reg, Register tmp_reg,

                           bool swap_reg_contains_mark,

                           Label& done, Label* slow_case = NULL,

                           BiasedLockingCounters* counters = NULL);

  void biased_locking_exit (Register obj_reg, Register temp_reg, Label& done);

  // Helper functions for statistics gathering.

  // Unconditional atomic increment.

  void atomic_incw(Register counter_addr, Register tmp, Register tmp2);

  void atomic_incw(Address counter_addr, Register tmp1, Register tmp2, Register tmp3) {

    lea(tmp1, counter_addr);

    atomic_incw(tmp1, tmp2, tmp3);

  }

  // Load Effective Address

  void lea(Register r, const Address &a) {

    InstructionMark im(this);

    code_section()->relocate(inst_mark(), a.rspec());

    a.lea(this, r);

  }

  void addmw(Address a, Register incr, Register scratch) {

    ldrw(scratch, a);

    addw(scratch, scratch, incr);

    strw(scratch, a);

  }

  // Add constant to memory word

  void addmw(Address a, int imm, Register scratch) {

    ldrw(scratch, a);

    if (imm > 0)

      addw(scratch, scratch, (unsigned)imm);

    else

      subw(scratch, scratch, (unsigned)-imm);

    strw(scratch, a);

  }

  void bind(Label& L) {

    Assembler::bind(L);

    code()->clear_last_insn();

  }

  void membar(Membar_mask_bits order_constraint);

  using Assembler::ldr;

  using Assembler::str;

  void ldr(Register Rx, const Address &adr);

  void ldrw(Register Rw, const Address &adr);

  void str(Register Rx, const Address &adr);

  void strw(Register Rx, const Address &adr);

  // Frame creation and destruction shared between JITs.

  void build_frame(int framesize);

  void remove_frame(int framesize);

  virtual void _call_Unimplemented(address call_site) {

    mov(rscratch2, call_site);

  }

#define call_Unimplemented() _call_Unimplemented((address)__PRETTY_FUNCTION__)

  // aliases defined in AARCH64 spec

  template<class T>

  inline void cmpw(Register Rd, T imm)  { subsw(zr, Rd, imm); }

  inline void cmp(Register Rd, unsigned char imm8)  { subs(zr, Rd, imm8); }

  inline void cmp(Register Rd, unsigned imm) __attribute__ ((deprecated));

  inline void cmnw(Register Rd, unsigned imm) { addsw(zr, Rd, imm); }

  inline void cmn(Register Rd, unsigned imm) { adds(zr, Rd, imm); }

  void cset(Register Rd, Assembler::Condition cond) {

    csinc(Rd, zr, zr, ~cond);

  }

  void csetw(Register Rd, Assembler::Condition cond) {

    csincw(Rd, zr, zr, ~cond);

  }

  void cneg(Register Rd, Register Rn, Assembler::Condition cond) {

    csneg(Rd, Rn, Rn, ~cond);

  }

  void cnegw(Register Rd, Register Rn, Assembler::Condition cond) {

    csnegw(Rd, Rn, Rn, ~cond);

  }

  inline void movw(Register Rd, Register Rn) {

    if (Rd == sp || Rn == sp) {

      addw(Rd, Rn, 0U);

    } else {

      orrw(Rd, zr, Rn);

    }

  }

  inline void mov(Register Rd, Register Rn) {

    assert(Rd != r31_sp && Rn != r31_sp, "should be");

    if (Rd == Rn) {

    } else if (Rd == sp || Rn == sp) {

      add(Rd, Rn, 0U);

    } else {

      orr(Rd, zr, Rn);

    }

  }

  inline void moviw(Register Rd, unsigned imm) { orrw(Rd, zr, imm); }

  inline void movi(Register Rd, unsigned imm) { orr(Rd, zr, imm); }

  inline void tstw(Register Rd, Register Rn) { andsw(zr, Rd, Rn); }

  inline void tst(Register Rd, Register Rn) { ands(zr, Rd, Rn); }

  inline void tstw(Register Rd, uint64_t imm) { andsw(zr, Rd, imm); }

  inline void tst(Register Rd, uint64_t imm) { ands(zr, Rd, imm); }

  inline void bfiw(Register Rd, Register Rn, unsigned lsb, unsigned width) {

    bfmw(Rd, Rn, ((32 - lsb) & 31), (width - 1));

  }

  inline void bfi(Register Rd, Register Rn, unsigned lsb, unsigned width) {

    bfm(Rd, Rn, ((64 - lsb) & 63), (width - 1));

  }

  inline void bfxilw(Register Rd, Register Rn, unsigned lsb, unsigned width) {

    bfmw(Rd, Rn, lsb, (lsb + width - 1));

  }

  inline void bfxil(Register Rd, Register Rn, unsigned lsb, unsigned width) {

    bfm(Rd, Rn, lsb , (lsb + width - 1));

  }

  inline void sbfizw(Register Rd, Register Rn, unsigned lsb, unsigned width) {

    sbfmw(Rd, Rn, ((32 - lsb) & 31), (width - 1));

  }

  inline void sbfiz(Register Rd, Register Rn, unsigned lsb, unsigned width) {

    sbfm(Rd, Rn, ((64 - lsb) & 63), (width - 1));

  }

  inline void sbfxw(Register Rd, Register Rn, unsigned lsb, unsigned width) {

    sbfmw(Rd, Rn, lsb, (lsb + width - 1));

  }

  inline void sbfx(Register Rd, Register Rn, unsigned lsb, unsigned width) {

    sbfm(Rd, Rn, lsb , (lsb + width - 1));

  }

  inline void ubfizw(Register Rd, Register Rn, unsigned lsb, unsigned width) {

    ubfmw(Rd, Rn, ((32 - lsb) & 31), (width - 1));

  }

  inline void ubfiz(Register Rd, Register Rn, unsigned lsb, unsigned width) {

    ubfm(Rd, Rn, ((64 - lsb) & 63), (width - 1));

  }

  inline void ubfxw(Register Rd, Register Rn, unsigned lsb, unsigned width) {

    ubfmw(Rd, Rn, lsb, (lsb + width - 1));

  }

  inline void ubfx(Register Rd, Register Rn, unsigned lsb, unsigned width) {

    ubfm(Rd, Rn, lsb , (lsb + width - 1));

  }

  inline void asrw(Register Rd, Register Rn, unsigned imm) {

    sbfmw(Rd, Rn, imm, 31);

  }

  inline void asr(Register Rd, Register Rn, unsigned imm) {

    sbfm(Rd, Rn, imm, 63);

  }

  inline void lslw(Register Rd, Register Rn, unsigned imm) {

    ubfmw(Rd, Rn, ((32 - imm) & 31), (31 - imm));

  }

  inline void lsl(Register Rd, Register Rn, unsigned imm) {

    ubfm(Rd, Rn, ((64 - imm) & 63), (63 - imm));

  }

  inline void lsrw(Register Rd, Register Rn, unsigned imm) {

    ubfmw(Rd, Rn, imm, 31);

  }

  inline void lsr(Register Rd, Register Rn, unsigned imm) {

    ubfm(Rd, Rn, imm, 63);

  }

  inline void rorw(Register Rd, Register Rn, unsigned imm) {

    extrw(Rd, Rn, Rn, imm);

  }

  inline void ror(Register Rd, Register Rn, unsigned imm) {

    extr(Rd, Rn, Rn, imm);

  }

  inline void sxtbw(Register Rd, Register Rn) {

    sbfmw(Rd, Rn, 0, 7);

  }

  inline void sxthw(Register Rd, Register Rn) {

    sbfmw(Rd, Rn, 0, 15);

  }

  inline void sxtb(Register Rd, Register Rn) {

    sbfm(Rd, Rn, 0, 7);

  }

  inline void sxth(Register Rd, Register Rn) {

    sbfm(Rd, Rn, 0, 15);

  }

  inline void sxtw(Register Rd, Register Rn) {

    sbfm(Rd, Rn, 0, 31);

  }

  inline void uxtbw(Register Rd, Register Rn) {

    ubfmw(Rd, Rn, 0, 7);

  }

  inline void uxthw(Register Rd, Register Rn) {

    ubfmw(Rd, Rn, 0, 15);

  }

  inline void uxtb(Register Rd, Register Rn) {

    ubfm(Rd, Rn, 0, 7);

  }

  inline void uxth(Register Rd, Register Rn) {

    ubfm(Rd, Rn, 0, 15);

  }

  inline void uxtw(Register Rd, Register Rn) {

    ubfm(Rd, Rn, 0, 31);

  }

  inline void cmnw(Register Rn, Register Rm) {

    addsw(zr, Rn, Rm);

  }

  inline void cmn(Register Rn, Register Rm) {

    adds(zr, Rn, Rm);

  }

  inline void cmpw(Register Rn, Register Rm) {

    subsw(zr, Rn, Rm);

  }

  inline void cmp(Register Rn, Register Rm) {

    subs(zr, Rn, Rm);

  }

  inline void negw(Register Rd, Register Rn) {

    subw(Rd, zr, Rn);

  }

  inline void neg(Register Rd, Register Rn) {

    sub(Rd, zr, Rn);

  }

  inline void negsw(Register Rd, Register Rn) {

    subsw(Rd, zr, Rn);

  }

  inline void negs(Register Rd, Register Rn) {

    subs(Rd, zr, Rn);

  }

  inline void cmnw(Register Rn, Register Rm, enum shift_kind kind, unsigned shift = 0) {

    addsw(zr, Rn, Rm, kind, shift);

  }

  inline void cmn(Register Rn, Register Rm, enum shift_kind kind, unsigned shift = 0) {

    adds(zr, Rn, Rm, kind, shift);

  }

  inline void cmpw(Register Rn, Register Rm, enum shift_kind kind, unsigned shift = 0) {

    subsw(zr, Rn, Rm, kind, shift);

  }

  inline void cmp(Register Rn, Register Rm, enum shift_kind kind, unsigned shift = 0) {

    subs(zr, Rn, Rm, kind, shift);

  }

  inline void negw(Register Rd, Register Rn, enum shift_kind kind, unsigned shift = 0) {

    subw(Rd, zr, Rn, kind, shift);

  }

  inline void neg(Register Rd, Register Rn, enum shift_kind kind, unsigned shift = 0) {

    sub(Rd, zr, Rn, kind, shift);

  }

  inline void negsw(Register Rd, Register Rn, enum shift_kind kind, unsigned shift = 0) {

    subsw(Rd, zr, Rn, kind, shift);

  }

  inline void negs(Register Rd, Register Rn, enum shift_kind kind, unsigned shift = 0) {

    subs(Rd, zr, Rn, kind, shift);

  }

  inline void mnegw(Register Rd, Register Rn, Register Rm) {

    msubw(Rd, Rn, Rm, zr);

  }

  inline void mneg(Register Rd, Register Rn, Register Rm) {

    msub(Rd, Rn, Rm, zr);

  }

  inline void mulw(Register Rd, Register Rn, Register Rm) {

    maddw(Rd, Rn, Rm, zr);

  }

  inline void mul(Register Rd, Register Rn, Register Rm) {

    madd(Rd, Rn, Rm, zr);

  }

  inline void smnegl(Register Rd, Register Rn, Register Rm) {

    smsubl(Rd, Rn, Rm, zr);

  }

  inline void smull(Register Rd, Register Rn, Register Rm) {

    smaddl(Rd, Rn, Rm, zr);

  }

  inline void umnegl(Register Rd, Register Rn, Register Rm) {

    umsubl(Rd, Rn, Rm, zr);

  }

  inline void umull(Register Rd, Register Rn, Register Rm) {

    umaddl(Rd, Rn, Rm, zr);

  }

#define WRAP(INSN)                                                            \

  void INSN(Register Rd, Register Rn, Register Rm, Register Ra) {             \

    if ((VM_Version::features() & VM_Version::CPU_A53MAC) && Ra != zr)        \

      nop();                                                                  \

    Assembler::INSN(Rd, Rn, Rm, Ra);                                          \

  }

  WRAP(madd) WRAP(msub) WRAP(maddw) WRAP(msubw)

  WRAP(smaddl) WRAP(smsubl) WRAP(umaddl) WRAP(umsubl)

#undef WRAP

  // macro assembly operations needed for aarch64

  // first two private routines for loading 32 bit or 64 bit constants

private:

  void mov_immediate64(Register dst, u_int64_t imm64);

  void mov_immediate32(Register dst, u_int32_t imm32);

  int push(unsigned int bitset, Register stack);

  int pop(unsigned int bitset, Register stack);

  int push_fp(unsigned int bitset, Register stack);

  int pop_fp(unsigned int bitset, Register stack);

  void mov(Register dst, Address a);

public:

  void push(RegSet regs, Register stack) { if (regs.bits()) push(regs.bits(), stack); }

  void pop(RegSet regs, Register stack) { if (regs.bits()) pop(regs.bits(), stack); }

  void push_fp(RegSet regs, Register stack) { if (regs.bits()) push_fp(regs.bits(), stack); }

  void pop_fp(RegSet regs, Register stack) { if (regs.bits()) pop_fp(regs.bits(), stack); }

  // Push and pop everything that might be clobbered by a native

  // runtime call except rscratch1 and rscratch2.  (They are always

  // scratch, so we don't have to protect them.)  Only save the lower

  // 64 bits of each vector register.

  void push_call_clobbered_registers();

  void pop_call_clobbered_registers();

  // now mov instructions for loading absolute addresses and 32 or

  // 64 bit integers

  inline void mov(Register dst, address addr)

  {

    mov_immediate64(dst, (u_int64_t)addr);

  }

  inline void mov(Register dst, u_int64_t imm64)

  {

    mov_immediate64(dst, imm64);

  }

  inline void movw(Register dst, u_int32_t imm32)

  {

    mov_immediate32(dst, imm32);

  }

  inline void mov(Register dst, long l)

  {

    mov(dst, (u_int64_t)l);

  }

  inline void mov(Register dst, int i)

  {

    mov(dst, (long)i);

  }

  void mov(Register dst, RegisterOrConstant src) {

    if (src.is_register())

      mov(dst, src.as_register());

    else

      mov(dst, src.as_constant());

  }

  void movptr(Register r, uintptr_t imm64);

  void mov(FloatRegister Vd, SIMD_Arrangement T, u_int32_t imm32);

  void mov(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn) {

    orr(Vd, T, Vn, Vn);

  }

public:

  // Generalized Test Bit And Branch, including a "far" variety which

  // spans more than 32KiB.

  void tbr(Condition cond, Register Rt, int bitpos, Label &dest, bool far = false) {

    assert(cond == EQ || cond == NE, "must be");

    if (far)

      cond = ~cond;

    void (Assembler::* branch)(Register Rt, int bitpos, Label &L);

    if (cond == Assembler::EQ)