/*

 * Copyright (c) 1997, 2017, Oracle and/or its affiliates. All rights reserved.

 * Copyright (c) 2014, 2015, Red Hat Inc. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include <sys/types.h>

#include "precompiled.hpp"

#include "jvm.h"

#include "asm/assembler.hpp"

#include "asm/assembler.inline.hpp"

#include "interpreter/interpreter.hpp"

#include "compiler/disassembler.hpp"

#include "memory/resourceArea.hpp"

#include "nativeInst_aarch64.hpp"

#include "oops/klass.inline.hpp"

#include "oops/oop.inline.hpp"

#include "opto/compile.hpp"

#include "opto/intrinsicnode.hpp"

#include "opto/node.hpp"

#include "runtime/biasedLocking.hpp"

#include "runtime/icache.hpp"

#include "runtime/interfaceSupport.hpp"

#include "runtime/sharedRuntime.hpp"

#include "runtime/thread.hpp"

#if INCLUDE_ALL_GCS

#include "gc/g1/g1CollectedHeap.inline.hpp"

#include "gc/g1/g1SATBCardTableModRefBS.hpp"

#include "gc/g1/heapRegion.hpp"

#endif

#ifdef PRODUCT

#define BLOCK_COMMENT(str) /* nothing */

#define STOP(error) stop(error)

#else

#define BLOCK_COMMENT(str) block_comment(str)

#define STOP(error) block_comment(error); stop(error)

#endif

#define BIND(label) bind(label); BLOCK_COMMENT(#label ":")

// Patch any kind of instruction; there may be several instructions.

// Return the total length (in bytes) of the instructions.

int MacroAssembler::pd_patch_instruction_size(address branch, address target) {

  int instructions = 1;

  assert((uint64_t)target < (1ul << 48), "48-bit overflow in address constant");

  long offset = (target - branch) >> 2;

  unsigned insn = *(unsigned*)branch;

  if ((Instruction_aarch64::extract(insn, 29, 24) & 0b111011) == 0b011000) {

    // Load register (literal)

    Instruction_aarch64::spatch(branch, 23, 5, offset);

  } else if (Instruction_aarch64::extract(insn, 30, 26) == 0b00101) {

    // Unconditional branch (immediate)

    Instruction_aarch64::spatch(branch, 25, 0, offset);

  } else if (Instruction_aarch64::extract(insn, 31, 25) == 0b0101010) {

    // Conditional branch (immediate)

    Instruction_aarch64::spatch(branch, 23, 5, offset);

  } else if (Instruction_aarch64::extract(insn, 30, 25) == 0b011010) {

    // Compare & branch (immediate)

    Instruction_aarch64::spatch(branch, 23, 5, offset);

  } else if (Instruction_aarch64::extract(insn, 30, 25) == 0b011011) {

    // Test & branch (immediate)

    Instruction_aarch64::spatch(branch, 18, 5, offset);

  } else if (Instruction_aarch64::extract(insn, 28, 24) == 0b10000) {

    // PC-rel. addressing

    offset = target-branch;

    int shift = Instruction_aarch64::extract(insn, 31, 31);

    if (shift) {

      u_int64_t dest = (u_int64_t)target;

      uint64_t pc_page = (uint64_t)branch >> 12;

      uint64_t adr_page = (uint64_t)target >> 12;

      unsigned offset_lo = dest & 0xfff;

      offset = adr_page - pc_page;

      // We handle 4 types of PC relative addressing

      //   1 - adrp    Rx, target_page

      //       ldr/str Ry, [Rx, #offset_in_page]

      //   2 - adrp    Rx, target_page

      //       add     Ry, Rx, #offset_in_page

      //   3 - adrp    Rx, target_page (page aligned reloc, offset == 0)

      //       movk    Rx, #imm16<<32

      //   4 - adrp    Rx, target_page (page aligned reloc, offset == 0)

      // In the first 3 cases we must check that Rx is the same in the adrp and the

      // subsequent ldr/str, add or movk instruction. Otherwise we could accidentally end

      // up treating a type 4 relocation as a type 1, 2 or 3 just because it happened

      // to be followed by a random unrelated ldr/str, add or movk instruction.

      //

      unsigned insn2 = ((unsigned*)branch)[1];

      if (Instruction_aarch64::extract(insn2, 29, 24) == 0b111001 &&

                Instruction_aarch64::extract(insn, 4, 0) ==

                        Instruction_aarch64::extract(insn2, 9, 5)) {

        // Load/store register (unsigned immediate)

        unsigned size = Instruction_aarch64::extract(insn2, 31, 30);

        Instruction_aarch64::patch(branch + sizeof (unsigned),

                                    21, 10, offset_lo >> size);

        guarantee(((dest >> size) << size) == dest, "misaligned target");

        instructions = 2;

      } else if (Instruction_aarch64::extract(insn2, 31, 22) == 0b1001000100 &&

                Instruction_aarch64::extract(insn, 4, 0) ==

                        Instruction_aarch64::extract(insn2, 4, 0)) {

        // add (immediate)

        Instruction_aarch64::patch(branch + sizeof (unsigned),

                                   21, 10, offset_lo);

        instructions = 2;

      } else if (Instruction_aarch64::extract(insn2, 31, 21) == 0b11110010110 &&

                   Instruction_aarch64::extract(insn, 4, 0) ==

                     Instruction_aarch64::extract(insn2, 4, 0)) {

        // movk #imm16<<32

        Instruction_aarch64::patch(branch + 4, 20, 5, (uint64_t)target >> 32);

        long dest = ((long)target & 0xffffffffL) | ((long)branch & 0xffff00000000L);

        long pc_page = (long)branch >> 12;

        long adr_page = (long)dest >> 12;

        offset = adr_page - pc_page;

        instructions = 2;

      }

    }

    int offset_lo = offset & 3;

    offset >>= 2;

    Instruction_aarch64::spatch(branch, 23, 5, offset);

    Instruction_aarch64::patch(branch, 30, 29, offset_lo);

  } else if (Instruction_aarch64::extract(insn, 31, 21) == 0b11010010100) {

    u_int64_t dest = (u_int64_t)target;

    // Move wide constant

    assert(nativeInstruction_at(branch+4)->is_movk(), "wrong insns in patch");

    assert(nativeInstruction_at(branch+8)->is_movk(), "wrong insns in patch");

    Instruction_aarch64::patch(branch, 20, 5, dest & 0xffff);

    Instruction_aarch64::patch(branch+4, 20, 5, (dest >>= 16) & 0xffff);

    Instruction_aarch64::patch(branch+8, 20, 5, (dest >>= 16) & 0xffff);

    assert(target_addr_for_insn(branch) == target, "should be");

    instructions = 3;

  } else if (Instruction_aarch64::extract(insn, 31, 22) == 0b1011100101 &&

             Instruction_aarch64::extract(insn, 4, 0) == 0b11111) {

    // nothing to do

    assert(target == 0, "did not expect to relocate target for polling page load");

  } else {

    ShouldNotReachHere();

  }

  return instructions * NativeInstruction::instruction_size;

}

int MacroAssembler::patch_oop(address insn_addr, address o) {

  int instructions;

  unsigned insn = *(unsigned*)insn_addr;

  assert(nativeInstruction_at(insn_addr+4)->is_movk(), "wrong insns in patch");

  // OOPs are either narrow (32 bits) or wide (48 bits).  We encode

  // narrow OOPs by setting the upper 16 bits in the first

  // instruction.

  if (Instruction_aarch64::extract(insn, 31, 21) == 0b11010010101) {

    // Move narrow OOP

    narrowOop n = oopDesc::encode_heap_oop((oop)o);

    Instruction_aarch64::patch(insn_addr, 20, 5, n >> 16);

    Instruction_aarch64::patch(insn_addr+4, 20, 5, n & 0xffff);

    instructions = 2;

  } else {

    // Move wide OOP

    assert(nativeInstruction_at(insn_addr+8)->is_movk(), "wrong insns in patch");

    uintptr_t dest = (uintptr_t)o;

    Instruction_aarch64::patch(insn_addr, 20, 5, dest & 0xffff);

    Instruction_aarch64::patch(insn_addr+4, 20, 5, (dest >>= 16) & 0xffff);

    Instruction_aarch64::patch(insn_addr+8, 20, 5, (dest >>= 16) & 0xffff);

    instructions = 3;

  }

  return instructions * NativeInstruction::instruction_size;

}

int MacroAssembler::patch_narrow_klass(address insn_addr, narrowKlass n) {

  // Metatdata pointers are either narrow (32 bits) or wide (48 bits).

  // We encode narrow ones by setting the upper 16 bits in the first

  // instruction.

  NativeInstruction *insn = nativeInstruction_at(insn_addr);

  assert(Instruction_aarch64::extract(insn->encoding(), 31, 21) == 0b11010010101 &&

         nativeInstruction_at(insn_addr+4)->is_movk(), "wrong insns in patch");

  Instruction_aarch64::patch(insn_addr, 20, 5, n >> 16);

  Instruction_aarch64::patch(insn_addr+4, 20, 5, n & 0xffff);

  return 2 * NativeInstruction::instruction_size;

}

address MacroAssembler::target_addr_for_insn(address insn_addr, unsigned insn) {

  long offset = 0;

  if ((Instruction_aarch64::extract(insn, 29, 24) & 0b011011) == 0b00011000) {

    // Load register (literal)

    offset = Instruction_aarch64::sextract(insn, 23, 5);

    return address(((uint64_t)insn_addr + (offset << 2)));

  } else if (Instruction_aarch64::extract(insn, 30, 26) == 0b00101) {

    // Unconditional branch (immediate)

    offset = Instruction_aarch64::sextract(insn, 25, 0);

  } else if (Instruction_aarch64::extract(insn, 31, 25) == 0b0101010) {

    // Conditional branch (immediate)

    offset = Instruction_aarch64::sextract(insn, 23, 5);

  } else if (Instruction_aarch64::extract(insn, 30, 25) == 0b011010) {

    // Compare & branch (immediate)

    offset = Instruction_aarch64::sextract(insn, 23, 5);

   } else if (Instruction_aarch64::extract(insn, 30, 25) == 0b011011) {

    // Test & branch (immediate)

    offset = Instruction_aarch64::sextract(insn, 18, 5);

  } else if (Instruction_aarch64::extract(insn, 28, 24) == 0b10000) {

    // PC-rel. addressing

    offset = Instruction_aarch64::extract(insn, 30, 29);

    offset |= Instruction_aarch64::sextract(insn, 23, 5) << 2;

    int shift = Instruction_aarch64::extract(insn, 31, 31) ? 12 : 0;

    if (shift) {

      offset <<= shift;

      uint64_t target_page = ((uint64_t)insn_addr) + offset;

      target_page &= ((uint64_t)-1) << shift;

      // Return the target address for the following sequences

      //   1 - adrp    Rx, target_page

      //       ldr/str Ry, [Rx, #offset_in_page]

      //   2 - adrp    Rx, target_page

      //       add     Ry, Rx, #offset_in_page

      //   3 - adrp    Rx, target_page (page aligned reloc, offset == 0)

      //       movk    Rx, #imm12<<32

      //   4 - adrp    Rx, target_page (page aligned reloc, offset == 0)

      //

      // In the first two cases  we check that the register is the same and

      // return the target_page + the offset within the page.

      // Otherwise we assume it is a page aligned relocation and return

      // the target page only.

      //

      unsigned insn2 = ((unsigned*)insn_addr)[1];

      if (Instruction_aarch64::extract(insn2, 29, 24) == 0b111001 &&

                Instruction_aarch64::extract(insn, 4, 0) ==

                        Instruction_aarch64::extract(insn2, 9, 5)) {

        // Load/store register (unsigned immediate)

        unsigned int byte_offset = Instruction_aarch64::extract(insn2, 21, 10);

        unsigned int size = Instruction_aarch64::extract(insn2, 31, 30);

        return address(target_page + (byte_offset << size));

      } else if (Instruction_aarch64::extract(insn2, 31, 22) == 0b1001000100 &&

                Instruction_aarch64::extract(insn, 4, 0) ==

                        Instruction_aarch64::extract(insn2, 4, 0)) {

        // add (immediate)

        unsigned int byte_offset = Instruction_aarch64::extract(insn2, 21, 10);

        return address(target_page + byte_offset);

      } else {

        if (Instruction_aarch64::extract(insn2, 31, 21) == 0b11110010110  &&

               Instruction_aarch64::extract(insn, 4, 0) ==

                 Instruction_aarch64::extract(insn2, 4, 0)) {

          target_page = (target_page & 0xffffffff) |

                         ((uint64_t)Instruction_aarch64::extract(insn2, 20, 5) << 32);

        }

        return (address)target_page;

      }

    } else {

      ShouldNotReachHere();

    }

  } else if (Instruction_aarch64::extract(insn, 31, 23) == 0b110100101) {

    u_int32_t *insns = (u_int32_t *)insn_addr;

    // Move wide constant: movz, movk, movk.  See movptr().

    assert(nativeInstruction_at(insns+1)->is_movk(), "wrong insns in patch");

    assert(nativeInstruction_at(insns+2)->is_movk(), "wrong insns in patch");

    return address(u_int64_t(Instruction_aarch64::extract(insns[0], 20, 5))

                   + (u_int64_t(Instruction_aarch64::extract(insns[1], 20, 5)) << 16)

                   + (u_int64_t(Instruction_aarch64::extract(insns[2], 20, 5)) << 32));

  } else if (Instruction_aarch64::extract(insn, 31, 22) == 0b1011100101 &&

             Instruction_aarch64::extract(insn, 4, 0) == 0b11111) {

    return 0;

  } else {

    ShouldNotReachHere();

  }

  return address(((uint64_t)insn_addr + (offset << 2)));

}

void MacroAssembler::serialize_memory(Register thread, Register tmp) {

  dsb(Assembler::SY);

}

void MacroAssembler::safepoint_poll(Label& slow_path) {

  if (SafepointMechanism::uses_thread_local_poll()) {

    ldr(rscratch1, Address(rthread, Thread::polling_page_offset()));

    tbnz(rscratch1, exact_log2(SafepointMechanism::poll_bit()), slow_path);

  } else {

    unsigned long offset;

    adrp(rscratch1, ExternalAddress(SafepointSynchronize::address_of_state()), offset);

    ldrw(rscratch1, Address(rscratch1, offset));

    assert(SafepointSynchronize::_not_synchronized == 0, "rewrite this code");

    cbnz(rscratch1, slow_path);

  }

}

// Just like safepoint_poll, but use an acquiring load for thread-

// local polling.

//

// We need an acquire here to ensure that any subsequent load of the

// global SafepointSynchronize::_state flag is ordered after this load

// of the local Thread::_polling page.  We don't want this poll to

// return false (i.e. not safepointing) and a later poll of the global

// SafepointSynchronize::_state spuriously to return true.

//

// This is to avoid a race when we're in a native->Java transition

// racing the code which wakes up from a safepoint.

//

void MacroAssembler::safepoint_poll_acquire(Label& slow_path) {

  if (SafepointMechanism::uses_thread_local_poll()) {

    lea(rscratch1, Address(rthread, Thread::polling_page_offset()));

    ldar(rscratch1, rscratch1);

    tbnz(rscratch1, exact_log2(SafepointMechanism::poll_bit()), slow_path);

  } else {

    safepoint_poll(slow_path);

  }

}

void MacroAssembler::reset_last_Java_frame(bool clear_fp) {

  // we must set sp to zero to clear frame

  str(zr, Address(rthread, JavaThread::last_Java_sp_offset()));

  // must clear fp, so that compiled frames are not confused; it is

  // possible that we need it only for debugging

  if (clear_fp) {

    str(zr, Address(rthread, JavaThread::last_Java_fp_offset()));

  }

  // Always clear the pc because it could have been set by make_walkable()

  str(zr, Address(rthread, JavaThread::last_Java_pc_offset()));

}

// Calls to C land

//

// When entering C land, the rfp, & resp of the last Java frame have to be recorded

// in the (thread-local) JavaThread object. When leaving C land, the last Java fp

// has to be reset to 0. This is required to allow proper stack traversal.

void MacroAssembler::set_last_Java_frame(Register last_java_sp,

                                         Register last_java_fp,

                                         Register last_java_pc,

                                         Register scratch) {

  if (last_java_pc->is_valid()) {

      str(last_java_pc, Address(rthread,

                                JavaThread::frame_anchor_offset()

                                + JavaFrameAnchor::last_Java_pc_offset()));

    }

  // determine last_java_sp register

  if (last_java_sp == sp) {

    mov(scratch, sp);

    last_java_sp = scratch;

  } else if (!last_java_sp->is_valid()) {

    last_java_sp = esp;

  }

  str(last_java_sp, Address(rthread, JavaThread::last_Java_sp_offset()));

  // last_java_fp is optional

  if (last_java_fp->is_valid()) {

    str(last_java_fp, Address(rthread, JavaThread::last_Java_fp_offset()));

  }

}

void MacroAssembler::set_last_Java_frame(Register last_java_sp,

                                         Register last_java_fp,

                                         address  last_java_pc,

                                         Register scratch) {

  if (last_java_pc != NULL) {

    adr(scratch, last_java_pc);

  } else {

    // FIXME: This is almost never correct.  We should delete all

    // cases of set_last_Java_frame with last_java_pc=NULL and use the

    // correct return address instead.

    adr(scratch, pc());

  }

  str(scratch, Address(rthread,

                       JavaThread::frame_anchor_offset()

                       + JavaFrameAnchor::last_Java_pc_offset()));

  set_last_Java_frame(last_java_sp, last_java_fp, noreg, scratch);

}

void MacroAssembler::set_last_Java_frame(Register last_java_sp,

                                         Register last_java_fp,

                                         Label &L,

                                         Register scratch) {

  if (L.is_bound()) {

    set_last_Java_frame(last_java_sp, last_java_fp, target(L), scratch);

  } else {

    InstructionMark im(this);

    L.add_patch_at(code(), locator());

    set_last_Java_frame(last_java_sp, last_java_fp, (address)NULL, scratch);

  }

}

void MacroAssembler::far_call(Address entry, CodeBuffer *cbuf, Register tmp) {

  assert(ReservedCodeCacheSize < 4*G, "branch out of range");

  assert(CodeCache::find_blob(entry.target()) != NULL,

         "destination of far call not found in code cache");

  if (far_branches()) {

    unsigned long offset;

    // We can use ADRP here because we know that the total size of

    // the code cache cannot exceed 2Gb.

    adrp(tmp, entry, offset);

    add(tmp, tmp, offset);

    if (cbuf) cbuf->set_insts_mark();

    blr(tmp);

  } else {

    if (cbuf) cbuf->set_insts_mark();

    bl(entry);

  }

}

void MacroAssembler::far_jump(Address entry, CodeBuffer *cbuf, Register tmp) {

  assert(ReservedCodeCacheSize < 4*G, "branch out of range");

  assert(CodeCache::find_blob(entry.target()) != NULL,

         "destination of far call not found in code cache");

  if (far_branches()) {

    unsigned long offset;

    // We can use ADRP here because we know that the total size of

    // the code cache cannot exceed 2Gb.

    adrp(tmp, entry, offset);

    add(tmp, tmp, offset);

    if (cbuf) cbuf->set_insts_mark();

    br(tmp);

  } else {

    if (cbuf) cbuf->set_insts_mark();

    b(entry);

  }

}

void MacroAssembler::reserved_stack_check() {

    // testing if reserved zone needs to be enabled

    Label no_reserved_zone_enabling;

    ldr(rscratch1, Address(rthread, JavaThread::reserved_stack_activation_offset()));

    cmp(sp, rscratch1);

    br(Assembler::LO, no_reserved_zone_enabling);

    enter();   // LR and FP are live.

    lea(rscratch1, CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone));

    mov(c_rarg0, rthread);

    blr(rscratch1);

    leave();

    // We have already removed our own frame.

    // throw_delayed_StackOverflowError will think that it's been

    // called by our caller.

    lea(rscratch1, RuntimeAddress(StubRoutines::throw_delayed_StackOverflowError_entry()));

    br(rscratch1);

    should_not_reach_here();

    bind(no_reserved_zone_enabling);

}

int MacroAssembler::biased_locking_enter(Register lock_reg,

                                         Register obj_reg,

                                         Register swap_reg,

                                         Register tmp_reg,

                                         bool swap_reg_contains_mark,

                                         Label& done,

                                         Label* slow_case,

                                         BiasedLockingCounters* counters) {

  assert(UseBiasedLocking, "why call this otherwise?");

  assert_different_registers(lock_reg, obj_reg, swap_reg);

  if (PrintBiasedLockingStatistics && counters == NULL)

    counters = BiasedLocking::counters();

  assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg, rscratch1, rscratch2, noreg);

  assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout");

  Address mark_addr      (obj_reg, oopDesc::mark_offset_in_bytes());

  Address klass_addr     (obj_reg, oopDesc::klass_offset_in_bytes());

  Address saved_mark_addr(lock_reg, 0);

  // Biased locking

  // See whether the lock is currently biased toward our thread and

  // whether the epoch is still valid

  // Note that the runtime guarantees sufficient alignment of JavaThread

  // pointers to allow age to be placed into low bits

  // First check to see whether biasing is even enabled for this object

  Label cas_label;

  int null_check_offset = -1;

  if (!swap_reg_contains_mark) {

    null_check_offset = offset();

    ldr(swap_reg, mark_addr);

  }

  andr(tmp_reg, swap_reg, markOopDesc::biased_lock_mask_in_place);

  cmp(tmp_reg, markOopDesc::biased_lock_pattern);

  br(Assembler::NE, cas_label);

  // The bias pattern is present in the object's header. Need to check

  // whether the bias owner and the epoch are both still current.

  load_prototype_header(tmp_reg, obj_reg);

  orr(tmp_reg, tmp_reg, rthread);

  eor(tmp_reg, swap_reg, tmp_reg);

  andr(tmp_reg, tmp_reg, ~((int) markOopDesc::age_mask_in_place));

  if (counters != NULL) {

    Label around;

    cbnz(tmp_reg, around);

    atomic_incw(Address((address)counters->biased_lock_entry_count_addr()), tmp_reg, rscratch1, rscratch2);

    b(done);

    bind(around);

  } else {

    cbz(tmp_reg, done);

  }

  Label try_revoke_bias;

  Label try_rebias;