/*

 * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved.

 * Copyright (c) 2014, 2015, Red Hat Inc. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include <sys/types.h>

#include "precompiled.hpp"

#include "asm/assembler.hpp"

#include "asm/assembler.inline.hpp"

#include "interpreter/interpreter.hpp"

#include "compiler/disassembler.hpp"

#include "memory/resourceArea.hpp"

#include "nativeInst_aarch64.hpp"

#include "oops/klass.inline.hpp"

#include "oops/oop.inline.hpp"

#include "opto/compile.hpp"

#include "opto/node.hpp"

#include "runtime/biasedLocking.hpp"

#include "runtime/icache.hpp"

#include "runtime/interfaceSupport.hpp"

#include "runtime/sharedRuntime.hpp"

#if INCLUDE_ALL_GCS

#include "gc/g1/g1CollectedHeap.inline.hpp"

#include "gc/g1/g1SATBCardTableModRefBS.hpp"

#include "gc/g1/heapRegion.hpp"

#endif

#ifdef PRODUCT

#define BLOCK_COMMENT(str) /* nothing */

#define STOP(error) stop(error)

#else

#define BLOCK_COMMENT(str) block_comment(str)

#define STOP(error) block_comment(error); stop(error)

#endif

#define BIND(label) bind(label); BLOCK_COMMENT(#label ":")

// Patch any kind of instruction; there may be several instructions.

// Return the total length (in bytes) of the instructions.

int MacroAssembler::pd_patch_instruction_size(address branch, address target) {

  int instructions = 1;

  assert((uint64_t)target < (1ul << 48), "48-bit overflow in address constant");

  long offset = (target - branch) >> 2;

  unsigned insn = *(unsigned*)branch;

  if ((Instruction_aarch64::extract(insn, 29, 24) & 0b111011) == 0b011000) {

    // Load register (literal)

    Instruction_aarch64::spatch(branch, 23, 5, offset);

  } else if (Instruction_aarch64::extract(insn, 30, 26) == 0b00101) {

    // Unconditional branch (immediate)

    Instruction_aarch64::spatch(branch, 25, 0, offset);

  } else if (Instruction_aarch64::extract(insn, 31, 25) == 0b0101010) {

    // Conditional branch (immediate)

    Instruction_aarch64::spatch(branch, 23, 5, offset);

  } else if (Instruction_aarch64::extract(insn, 30, 25) == 0b011010) {

    // Compare & branch (immediate)

    Instruction_aarch64::spatch(branch, 23, 5, offset);

  } else if (Instruction_aarch64::extract(insn, 30, 25) == 0b011011) {

    // Test & branch (immediate)

    Instruction_aarch64::spatch(branch, 18, 5, offset);

  } else if (Instruction_aarch64::extract(insn, 28, 24) == 0b10000) {

    // PC-rel. addressing

    offset = target-branch;

    int shift = Instruction_aarch64::extract(insn, 31, 31);

    if (shift) {

      u_int64_t dest = (u_int64_t)target;

      uint64_t pc_page = (uint64_t)branch >> 12;

      uint64_t adr_page = (uint64_t)target >> 12;

      unsigned offset_lo = dest & 0xfff;

      offset = adr_page - pc_page;

      // We handle 4 types of PC relative addressing

      //   1 - adrp    Rx, target_page

      //       ldr/str Ry, [Rx, #offset_in_page]

      //   2 - adrp    Rx, target_page

      //       add     Ry, Rx, #offset_in_page

      //   3 - adrp    Rx, target_page (page aligned reloc, offset == 0)

      //       movk    Rx, #imm16<<32

      //   4 - adrp    Rx, target_page (page aligned reloc, offset == 0)

      // In the first 3 cases we must check that Rx is the same in the adrp and the

      // subsequent ldr/str, add or movk instruction. Otherwise we could accidentally end

      // up treating a type 4 relocation as a type 1, 2 or 3 just because it happened

      // to be followed by a random unrelated ldr/str, add or movk instruction.

      //

      unsigned insn2 = ((unsigned*)branch)[1];

      if (Instruction_aarch64::extract(insn2, 29, 24) == 0b111001 &&

                Instruction_aarch64::extract(insn, 4, 0) ==

                        Instruction_aarch64::extract(insn2, 9, 5)) {

        // Load/store register (unsigned immediate)

        unsigned size = Instruction_aarch64::extract(insn2, 31, 30);

        Instruction_aarch64::patch(branch + sizeof (unsigned),

                                    21, 10, offset_lo >> size);

        guarantee(((dest >> size) << size) == dest, "misaligned target");

        instructions = 2;

      } else if (Instruction_aarch64::extract(insn2, 31, 22) == 0b1001000100 &&

                Instruction_aarch64::extract(insn, 4, 0) ==

                        Instruction_aarch64::extract(insn2, 4, 0)) {

        // add (immediate)

        Instruction_aarch64::patch(branch + sizeof (unsigned),

                                   21, 10, offset_lo);

        instructions = 2;

      } else if (Instruction_aarch64::extract(insn2, 31, 21) == 0b11110010110 &&

                   Instruction_aarch64::extract(insn, 4, 0) ==

                     Instruction_aarch64::extract(insn2, 4, 0)) {

        // movk #imm16<<32

        Instruction_aarch64::patch(branch + 4, 20, 5, (uint64_t)target >> 32);

        offset &= (1<<20)-1;

        instructions = 2;

      }

    }

    int offset_lo = offset & 3;

    offset >>= 2;

    Instruction_aarch64::spatch(branch, 23, 5, offset);

    Instruction_aarch64::patch(branch, 30, 29, offset_lo);

  } else if (Instruction_aarch64::extract(insn, 31, 21) == 0b11010010100) {

    u_int64_t dest = (u_int64_t)target;

    // Move wide constant

    assert(nativeInstruction_at(branch+4)->is_movk(), "wrong insns in patch");

    assert(nativeInstruction_at(branch+8)->is_movk(), "wrong insns in patch");

    Instruction_aarch64::patch(branch, 20, 5, dest & 0xffff);

    Instruction_aarch64::patch(branch+4, 20, 5, (dest >>= 16) & 0xffff);

    Instruction_aarch64::patch(branch+8, 20, 5, (dest >>= 16) & 0xffff);

    assert(target_addr_for_insn(branch) == target, "should be");

    instructions = 3;

  } else if (Instruction_aarch64::extract(insn, 31, 22) == 0b1011100101 &&

             Instruction_aarch64::extract(insn, 4, 0) == 0b11111) {

    // nothing to do

    assert(target == 0, "did not expect to relocate target for polling page load");

  } else {

    ShouldNotReachHere();

  }

  return instructions * NativeInstruction::instruction_size;

}

int MacroAssembler::patch_oop(address insn_addr, address o) {

  int instructions;

  unsigned insn = *(unsigned*)insn_addr;

  assert(nativeInstruction_at(insn_addr+4)->is_movk(), "wrong insns in patch");

  // OOPs are either narrow (32 bits) or wide (48 bits).  We encode

  // narrow OOPs by setting the upper 16 bits in the first

  // instruction.

  if (Instruction_aarch64::extract(insn, 31, 21) == 0b11010010101) {

    // Move narrow OOP

    narrowOop n = oopDesc::encode_heap_oop((oop)o);

    Instruction_aarch64::patch(insn_addr, 20, 5, n >> 16);

    Instruction_aarch64::patch(insn_addr+4, 20, 5, n & 0xffff);

    instructions = 2;

  } else {

    // Move wide OOP

    assert(nativeInstruction_at(insn_addr+8)->is_movk(), "wrong insns in patch");

    uintptr_t dest = (uintptr_t)o;

    Instruction_aarch64::patch(insn_addr, 20, 5, dest & 0xffff);

    Instruction_aarch64::patch(insn_addr+4, 20, 5, (dest >>= 16) & 0xffff);

    Instruction_aarch64::patch(insn_addr+8, 20, 5, (dest >>= 16) & 0xffff);

    instructions = 3;

  }

  return instructions * NativeInstruction::instruction_size;

}

address MacroAssembler::target_addr_for_insn(address insn_addr, unsigned insn) {

  long offset = 0;

  if ((Instruction_aarch64::extract(insn, 29, 24) & 0b011011) == 0b00011000) {

    // Load register (literal)

    offset = Instruction_aarch64::sextract(insn, 23, 5);

    return address(((uint64_t)insn_addr + (offset << 2)));

  } else if (Instruction_aarch64::extract(insn, 30, 26) == 0b00101) {

    // Unconditional branch (immediate)

    offset = Instruction_aarch64::sextract(insn, 25, 0);

  } else if (Instruction_aarch64::extract(insn, 31, 25) == 0b0101010) {

    // Conditional branch (immediate)

    offset = Instruction_aarch64::sextract(insn, 23, 5);

  } else if (Instruction_aarch64::extract(insn, 30, 25) == 0b011010) {

    // Compare & branch (immediate)

    offset = Instruction_aarch64::sextract(insn, 23, 5);

   } else if (Instruction_aarch64::extract(insn, 30, 25) == 0b011011) {

    // Test & branch (immediate)

    offset = Instruction_aarch64::sextract(insn, 18, 5);

  } else if (Instruction_aarch64::extract(insn, 28, 24) == 0b10000) {

    // PC-rel. addressing

    offset = Instruction_aarch64::extract(insn, 30, 29);

    offset |= Instruction_aarch64::sextract(insn, 23, 5) << 2;

    int shift = Instruction_aarch64::extract(insn, 31, 31) ? 12 : 0;

    if (shift) {

      offset <<= shift;

      uint64_t target_page = ((uint64_t)insn_addr) + offset;

      target_page &= ((uint64_t)-1) << shift;

      // Return the target address for the following sequences

      //   1 - adrp    Rx, target_page

      //       ldr/str Ry, [Rx, #offset_in_page]

      //   2 - adrp    Rx, target_page

      //       add     Ry, Rx, #offset_in_page

      //   3 - adrp    Rx, target_page (page aligned reloc, offset == 0)

      //       movk    Rx, #imm12<<32

      //   4 - adrp    Rx, target_page (page aligned reloc, offset == 0)

      //

      // In the first two cases  we check that the register is the same and

      // return the target_page + the offset within the page.

      // Otherwise we assume it is a page aligned relocation and return

      // the target page only.

      //

      unsigned insn2 = ((unsigned*)insn_addr)[1];

      if (Instruction_aarch64::extract(insn2, 29, 24) == 0b111001 &&

                Instruction_aarch64::extract(insn, 4, 0) ==

                        Instruction_aarch64::extract(insn2, 9, 5)) {

        // Load/store register (unsigned immediate)

        unsigned int byte_offset = Instruction_aarch64::extract(insn2, 21, 10);

        unsigned int size = Instruction_aarch64::extract(insn2, 31, 30);

        return address(target_page + (byte_offset << size));

      } else if (Instruction_aarch64::extract(insn2, 31, 22) == 0b1001000100 &&

                Instruction_aarch64::extract(insn, 4, 0) ==

                        Instruction_aarch64::extract(insn2, 4, 0)) {

        // add (immediate)

        unsigned int byte_offset = Instruction_aarch64::extract(insn2, 21, 10);

        return address(target_page + byte_offset);

      } else {

        if (Instruction_aarch64::extract(insn2, 31, 21) == 0b11110010110  &&

               Instruction_aarch64::extract(insn, 4, 0) ==

                 Instruction_aarch64::extract(insn2, 4, 0)) {

          target_page = (target_page & 0xffffffff) |

                         ((uint64_t)Instruction_aarch64::extract(insn2, 20, 5) << 32);

        }

        return (address)target_page;

      }

    } else {

      ShouldNotReachHere();

    }

  } else if (Instruction_aarch64::extract(insn, 31, 23) == 0b110100101) {

    u_int32_t *insns = (u_int32_t *)insn_addr;

    // Move wide constant: movz, movk, movk.  See movptr().

    assert(nativeInstruction_at(insns+1)->is_movk(), "wrong insns in patch");

    assert(nativeInstruction_at(insns+2)->is_movk(), "wrong insns in patch");

    return address(u_int64_t(Instruction_aarch64::extract(insns[0], 20, 5))

                   + (u_int64_t(Instruction_aarch64::extract(insns[1], 20, 5)) << 16)

                   + (u_int64_t(Instruction_aarch64::extract(insns[2], 20, 5)) << 32));

  } else if (Instruction_aarch64::extract(insn, 31, 22) == 0b1011100101 &&

             Instruction_aarch64::extract(insn, 4, 0) == 0b11111) {

    return 0;

  } else {

    ShouldNotReachHere();

  }

  return address(((uint64_t)insn_addr + (offset << 2)));

}

void MacroAssembler::serialize_memory(Register thread, Register tmp) {

  dsb(Assembler::SY);

}

void MacroAssembler::reset_last_Java_frame(bool clear_fp,

                                           bool clear_pc) {

  // we must set sp to zero to clear frame

  str(zr, Address(rthread, JavaThread::last_Java_sp_offset()));

  // must clear fp, so that compiled frames are not confused; it is

  // possible that we need it only for debugging

  if (clear_fp) {

    str(zr, Address(rthread, JavaThread::last_Java_fp_offset()));

  }

  if (clear_pc) {

    str(zr, Address(rthread, JavaThread::last_Java_pc_offset()));

  }

}

// Calls to C land

//

// When entering C land, the rfp, & resp of the last Java frame have to be recorded

// in the (thread-local) JavaThread object. When leaving C land, the last Java fp

// has to be reset to 0. This is required to allow proper stack traversal.

void MacroAssembler::set_last_Java_frame(Register last_java_sp,

                                         Register last_java_fp,

                                         Register last_java_pc,

                                         Register scratch) {

  if (last_java_pc->is_valid()) {

      str(last_java_pc, Address(rthread,

                                JavaThread::frame_anchor_offset()

                                + JavaFrameAnchor::last_Java_pc_offset()));

    }

  // determine last_java_sp register

  if (last_java_sp == sp) {

    mov(scratch, sp);

    last_java_sp = scratch;

  } else if (!last_java_sp->is_valid()) {

    last_java_sp = esp;

  }

  str(last_java_sp, Address(rthread, JavaThread::last_Java_sp_offset()));

  // last_java_fp is optional

  if (last_java_fp->is_valid()) {

    str(last_java_fp, Address(rthread, JavaThread::last_Java_fp_offset()));

  }

}

void MacroAssembler::set_last_Java_frame(Register last_java_sp,

                                         Register last_java_fp,

                                         address  last_java_pc,

                                         Register scratch) {

  if (last_java_pc != NULL) {

    adr(scratch, last_java_pc);

  } else {

    // FIXME: This is almost never correct.  We should delete all

    // cases of set_last_Java_frame with last_java_pc=NULL and use the

    // correct return address instead.

    adr(scratch, pc());

  }

  str(scratch, Address(rthread,

                       JavaThread::frame_anchor_offset()

                       + JavaFrameAnchor::last_Java_pc_offset()));

  set_last_Java_frame(last_java_sp, last_java_fp, noreg, scratch);

}

void MacroAssembler::set_last_Java_frame(Register last_java_sp,

                                         Register last_java_fp,

                                         Label &L,

                                         Register scratch) {

  if (L.is_bound()) {

    set_last_Java_frame(last_java_sp, last_java_fp, target(L), scratch);

  } else {

    InstructionMark im(this);

    L.add_patch_at(code(), locator());

    set_last_Java_frame(last_java_sp, last_java_fp, (address)NULL, scratch);

  }

}

void MacroAssembler::far_call(Address entry, CodeBuffer *cbuf, Register tmp) {

  assert(ReservedCodeCacheSize < 4*G, "branch out of range");

  assert(CodeCache::find_blob(entry.target()) != NULL,

         "destination of far call not found in code cache");

  if (far_branches()) {

    unsigned long offset;

    // We can use ADRP here because we know that the total size of

    // the code cache cannot exceed 2Gb.

    adrp(tmp, entry, offset);

    add(tmp, tmp, offset);

    if (cbuf) cbuf->set_insts_mark();

    blr(tmp);

  } else {

    if (cbuf) cbuf->set_insts_mark();

    bl(entry);

  }

}

void MacroAssembler::far_jump(Address entry, CodeBuffer *cbuf, Register tmp) {

  assert(ReservedCodeCacheSize < 4*G, "branch out of range");

  assert(CodeCache::find_blob(entry.target()) != NULL,

         "destination of far call not found in code cache");

  if (far_branches()) {

    unsigned long offset;

    // We can use ADRP here because we know that the total size of

    // the code cache cannot exceed 2Gb.

    adrp(tmp, entry, offset);

    add(tmp, tmp, offset);

    if (cbuf) cbuf->set_insts_mark();

    br(tmp);

  } else {

    if (cbuf) cbuf->set_insts_mark();

    b(entry);

  }

}

int MacroAssembler::biased_locking_enter(Register lock_reg,

                                         Register obj_reg,

                                         Register swap_reg,

                                         Register tmp_reg,

                                         bool swap_reg_contains_mark,

                                         Label& done,

                                         Label* slow_case,

                                         BiasedLockingCounters* counters) {

  assert(UseBiasedLocking, "why call this otherwise?");

  assert_different_registers(lock_reg, obj_reg, swap_reg);

  if (PrintBiasedLockingStatistics && counters == NULL)

    counters = BiasedLocking::counters();

  assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg, rscratch1, rscratch2, noreg);

  assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout");

  Address mark_addr      (obj_reg, oopDesc::mark_offset_in_bytes());

  Address klass_addr     (obj_reg, oopDesc::klass_offset_in_bytes());

  Address saved_mark_addr(lock_reg, 0);

  // Biased locking

  // See whether the lock is currently biased toward our thread and

  // whether the epoch is still valid

  // Note that the runtime guarantees sufficient alignment of JavaThread

  // pointers to allow age to be placed into low bits

  // First check to see whether biasing is even enabled for this object

  Label cas_label;

  int null_check_offset = -1;

  if (!swap_reg_contains_mark) {

    null_check_offset = offset();

    ldr(swap_reg, mark_addr);

  }

  andr(tmp_reg, swap_reg, markOopDesc::biased_lock_mask_in_place);

  cmp(tmp_reg, markOopDesc::biased_lock_pattern);

  br(Assembler::NE, cas_label);

  // The bias pattern is present in the object's header. Need to check

  // whether the bias owner and the epoch are both still current.

  load_prototype_header(tmp_reg, obj_reg);

  orr(tmp_reg, tmp_reg, rthread);

  eor(tmp_reg, swap_reg, tmp_reg);

  andr(tmp_reg, tmp_reg, ~((int) markOopDesc::age_mask_in_place));

  if (counters != NULL) {

    Label around;

    cbnz(tmp_reg, around);

    atomic_incw(Address((address)counters->biased_lock_entry_count_addr()), tmp_reg, rscratch1, rscratch2);

    b(done);

    bind(around);

  } else {

    cbz(tmp_reg, done);

  }

  Label try_revoke_bias;

  Label try_rebias;

  // At this point we know that the header has the bias pattern and

  // that we are not the bias owner in the current epoch. We need to

  // figure out more details about the state of the header in order to

  // know what operations can be legally performed on the object's

  // header.

  // If the low three bits in the xor result aren't clear, that means

  // the prototype header is no longer biased and we have to revoke

  // the bias on this object.

  andr(rscratch1, tmp_reg, markOopDesc::biased_lock_mask_in_place);

  cbnz(rscratch1, try_revoke_bias);

  // Biasing is still enabled for this data type. See whether the

  // epoch of the current bias is still valid, meaning that the epoch

  // bits of the mark word are equal to the epoch bits of the

  // prototype header. (Note that the prototype header's epoch bits

  // only change at a safepoint.) If not, attempt to rebias the object

  // toward the current thread. Note that we must be absolutely sure

  // that the current epoch is invalid in order to do this because

  // otherwise the manipulations it performs on the mark word are

  // illegal.

  andr(rscratch1, tmp_reg, markOopDesc::epoch_mask_in_place);

  cbnz(rscratch1, try_rebias);

  // The epoch of the current bias is still valid but we know nothing

  // about the owner; it might be set or it might be clear. Try to

  // acquire the bias of the object using an atomic operation. If this

  // fails we will go in to the runtime to revoke the object's bias.

  // Note that we first construct the presumed unbiased header so we

  // don't accidentally blow away another thread's valid bias.

  {

    Label here;

    mov(rscratch1, markOopDesc::biased_lock_mask_in_place | markOopDesc::age_mask_in_place | markOopDesc::epoch_mask_in_place);

    andr(swap_reg, swap_reg, rscratch1);

    orr(tmp_reg, swap_reg, rthread);

    cmpxchgptr(swap_reg, tmp_reg, obj_reg, rscratch1, here, slow_case);

    // If the biasing toward our thread failed, this means that

    // another thread succeeded in biasing it toward itself and we

    // need to revoke that bias. The revocation will occur in the

    // interpreter runtime in the slow case.

    bind(here);

    if (counters != NULL) {

      atomic_incw(Address((address)counters->anonymously_biased_lock_entry_count_addr()),

                  tmp_reg, rscratch1, rscratch2);

    }

  }

  b(done);

  bind(try_rebias);

  // At this point we know the epoch has expired, meaning that the

  // current "bias owner", if any, is actually invalid. Under these

  // circumstances _only_, we are allowed to use the current header's

  // value as the comparison value when doing the cas to acquire the

  // bias in the current epoch. In other words, we allow transfer of

  // the bias from one thread to another directly in this situation.

  //

  // FIXME: due to a lack of registers we currently blow away the age

  // bits in this situation. Should attempt to preserve them.

  {

    Label here;

    load_prototype_header(tmp_reg, obj_reg);

    orr(tmp_reg, rthread, tmp_reg);

    cmpxchgptr(swap_reg, tmp_reg, obj_reg, rscratch1, here, slow_case);

    // If the biasing toward our thread failed, then another thread

    // succeeded in biasing it toward itself and we need to revoke that

    // bias. The revocation will occur in the runtime in the slow case.

    bind(here);

    if (counters != NULL) {

      atomic_incw(Address((address)counters->rebiased_lock_entry_count_addr()),

                  tmp_reg, rscratch1, rscratch2);

    }

  }

  b(done);

  bind(try_revoke_bias);

  // The prototype mark in the klass doesn't have the bias bit set any

  // more, indicating that objects of this data type are not supposed

  // to be biased any more. We are going to try to reset the mark of

  // this object to the prototype value and fall through to the

  // CAS-based locking scheme. Note that if our CAS fails, it means

  // that another thread raced us for the privilege of revoking the

  // bias of this particular object, so it's okay to continue in the