/*

 * Copyright (c) 1999, 2018, Oracle and/or its affiliates. All rights reserved.

 * Copyright (c) 2014, Red Hat Inc. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "asm/assembler.hpp"

#include "c1/c1_CodeStubs.hpp"

#include "c1/c1_Defs.hpp"

#include "c1/c1_MacroAssembler.hpp"

#include "c1/c1_Runtime1.hpp"

#include "compiler/disassembler.hpp"

#include "gc/shared/cardTable.hpp"

#include "interpreter/interpreter.hpp"

#include "nativeInst_aarch64.hpp"

#include "oops/compiledICHolder.hpp"

#include "oops/oop.inline.hpp"

#include "prims/jvmtiExport.hpp"

#include "register_aarch64.hpp"

#include "runtime/sharedRuntime.hpp"

#include "runtime/signature.hpp"

#include "runtime/vframe.hpp"

#include "runtime/vframeArray.hpp"

#include "vmreg_aarch64.inline.hpp"

#if INCLUDE_ALL_GCS

#include "gc/g1/g1BarrierSet.hpp"

#include "gc/g1/g1CardTable.hpp"

#endif

// Implementation of StubAssembler

int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, int args_size) {

  // setup registers

  assert(!(oop_result1->is_valid() || metadata_result->is_valid()) || oop_result1 != metadata_result, "registers must be different");

  assert(oop_result1 != rthread && metadata_result != rthread, "registers must be different");

  assert(args_size >= 0, "illegal args_size");

  bool align_stack = false;

  mov(c_rarg0, rthread);

  set_num_rt_args(0); // Nothing on stack

  Label retaddr;

  set_last_Java_frame(sp, rfp, retaddr, rscratch1);

  // do the call

  lea(rscratch1, RuntimeAddress(entry));

  blrt(rscratch1, args_size + 1, 8, 1);

  bind(retaddr);

  int call_offset = offset();

  // verify callee-saved register

#ifdef ASSERT

  push(r0, sp);

  { Label L;

    get_thread(r0);

    cmp(rthread, r0);

    br(Assembler::EQ, L);

    stop("StubAssembler::call_RT: rthread not callee saved?");

    bind(L);

  }

  pop(r0, sp);

#endif

  reset_last_Java_frame(true);

  maybe_isb();

  // check for pending exceptions

  { Label L;

    // check for pending exceptions (java_thread is set upon return)

    ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));

    cbz(rscratch1, L);

    // exception pending => remove activation and forward to exception handler

    // make sure that the vm_results are cleared

    if (oop_result1->is_valid()) {

      str(zr, Address(rthread, JavaThread::vm_result_offset()));

    }

    if (metadata_result->is_valid()) {

      str(zr, Address(rthread, JavaThread::vm_result_2_offset()));

    }

    if (frame_size() == no_frame_size) {

      leave();

      far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));

    } else if (_stub_id == Runtime1::forward_exception_id) {

      should_not_reach_here();

    } else {

      far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::forward_exception_id)));

    }

    bind(L);

  }

  // get oop results if there are any and reset the values in the thread

  if (oop_result1->is_valid()) {

    get_vm_result(oop_result1, rthread);

  }

  if (metadata_result->is_valid()) {

    get_vm_result_2(metadata_result, rthread);

  }

  return call_offset;

}

int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1) {

  mov(c_rarg1, arg1);

  return call_RT(oop_result1, metadata_result, entry, 1);

}

int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2) {

  if (c_rarg1 == arg2) {

    if (c_rarg2 == arg1) {

      mov(rscratch1, arg1);

      mov(arg1, arg2);

      mov(arg2, rscratch1);

    } else {

      mov(c_rarg2, arg2);

      mov(c_rarg1, arg1);

    }

  } else {

    mov(c_rarg1, arg1);

    mov(c_rarg2, arg2);

  }

  return call_RT(oop_result1, metadata_result, entry, 2);

}

int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2, Register arg3) {

  // if there is any conflict use the stack

  if (arg1 == c_rarg2 || arg1 == c_rarg3 ||

      arg2 == c_rarg1 || arg1 == c_rarg3 ||

      arg3 == c_rarg1 || arg1 == c_rarg2) {

    stp(arg3, arg2, Address(pre(sp, 2 * wordSize)));

    stp(arg1, zr, Address(pre(sp, -2 * wordSize)));

    ldp(c_rarg1, zr, Address(post(sp, 2 * wordSize)));

    ldp(c_rarg3, c_rarg2, Address(post(sp, 2 * wordSize)));

  } else {

    mov(c_rarg1, arg1);

    mov(c_rarg2, arg2);

    mov(c_rarg3, arg3);

  }

  return call_RT(oop_result1, metadata_result, entry, 3);

}

// Implementation of StubFrame

class StubFrame: public StackObj {

 private:

  StubAssembler* _sasm;

 public:

  StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments);

  void load_argument(int offset_in_words, Register reg);

  ~StubFrame();

};;

#define __ _sasm->

StubFrame::StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments) {

  _sasm = sasm;

  __ set_info(name, must_gc_arguments);

  __ enter();

}

// load parameters that were stored with LIR_Assembler::store_parameter

// Note: offsets for store_parameter and load_argument must match

void StubFrame::load_argument(int offset_in_words, Register reg) {

  // rbp, + 0: link

  //     + 1: return address

  //     + 2: argument with offset 0

  //     + 3: argument with offset 1

  //     + 4: ...

  __ ldr(reg, Address(rfp, (offset_in_words + 2) * BytesPerWord));

}

StubFrame::~StubFrame() {

  __ leave();

  __ ret(lr);

}

#undef __

// Implementation of Runtime1

#define __ sasm->

const int float_regs_as_doubles_size_in_slots = pd_nof_fpu_regs_frame_map * 2;

// Stack layout for saving/restoring  all the registers needed during a runtime

// call (this includes deoptimization)

// Note: note that users of this frame may well have arguments to some runtime

// while these values are on the stack. These positions neglect those arguments

// but the code in save_live_registers will take the argument count into

// account.

//

enum reg_save_layout {

  reg_save_frame_size = 32 /* float */ + 32 /* integer */

};

// Save off registers which might be killed by calls into the runtime.

// Tries to smart of about FP registers.  In particular we separate

// saving and describing the FPU registers for deoptimization since we

// have to save the FPU registers twice if we describe them.  The

// deopt blob is the only thing which needs to describe FPU registers.

// In all other cases it should be sufficient to simply save their

// current value.

static int cpu_reg_save_offsets[FrameMap::nof_cpu_regs];

static int fpu_reg_save_offsets[FrameMap::nof_fpu_regs];

static int reg_save_size_in_words;

static int frame_size_in_bytes = -1;

static OopMap* generate_oop_map(StubAssembler* sasm, bool save_fpu_registers) {

  int frame_size_in_bytes = reg_save_frame_size * BytesPerWord;

  sasm->set_frame_size(frame_size_in_bytes / BytesPerWord);

  int frame_size_in_slots = frame_size_in_bytes / sizeof(jint);

  OopMap* oop_map = new OopMap(frame_size_in_slots, 0);

  for (int i = 0; i < FrameMap::nof_cpu_regs; i++) {

    Register r = as_Register(i);

    if (i <= 18 && i != rscratch1->encoding() && i != rscratch2->encoding()) {

      int sp_offset = cpu_reg_save_offsets[i];

      oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset),

                                r->as_VMReg());

    }

  }

  if (save_fpu_registers) {

    for (int i = 0; i < FrameMap::nof_fpu_regs; i++) {

      FloatRegister r = as_FloatRegister(i);

      {

        int sp_offset = fpu_reg_save_offsets[i];

        oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset),

                                  r->as_VMReg());

      }

    }

  }

  return oop_map;

}

static OopMap* save_live_registers(StubAssembler* sasm,

                                   bool save_fpu_registers = true) {

  __ block_comment("save_live_registers");

  __ push(RegSet::range(r0, r29), sp);         // integer registers except lr & sp

  if (save_fpu_registers) {

    for (int i = 30; i >= 0; i -= 2)

      __ stpd(as_FloatRegister(i), as_FloatRegister(i+1),

              Address(__ pre(sp, -2 * wordSize)));

  } else {

    __ add(sp, sp, -32 * wordSize);

  }

  return generate_oop_map(sasm, save_fpu_registers);

}

static void restore_live_registers(StubAssembler* sasm, bool restore_fpu_registers = true) {

  if (restore_fpu_registers) {

    for (int i = 0; i < 32; i += 2)

      __ ldpd(as_FloatRegister(i), as_FloatRegister(i+1),

              Address(__ post(sp, 2 * wordSize)));

  } else {

    __ add(sp, sp, 32 * wordSize);

  }

  __ pop(RegSet::range(r0, r29), sp);

}

static void restore_live_registers_except_r0(StubAssembler* sasm, bool restore_fpu_registers = true)  {

  if (restore_fpu_registers) {

    for (int i = 0; i < 32; i += 2)

      __ ldpd(as_FloatRegister(i), as_FloatRegister(i+1),

              Address(__ post(sp, 2 * wordSize)));

  } else {

    __ add(sp, sp, 32 * wordSize);

  }

  __ ldp(zr, r1, Address(__ post(sp, 16)));

  __ pop(RegSet::range(r2, r29), sp);

}

void Runtime1::initialize_pd() {

  int i;

  int sp_offset = 0;

  // all float registers are saved explicitly

  assert(FrameMap::nof_fpu_regs == 32, "double registers not handled here");

  for (i = 0; i < FrameMap::nof_fpu_regs; i++) {

    fpu_reg_save_offsets[i] = sp_offset;

    sp_offset += 2;   // SP offsets are in halfwords

  }

  for (i = 0; i < FrameMap::nof_cpu_regs; i++) {

    Register r = as_Register(i);

    cpu_reg_save_offsets[i] = sp_offset;

    sp_offset += 2;   // SP offsets are in halfwords

  }

}

// target: the entry point of the method that creates and posts the exception oop

// has_argument: true if the exception needs an argument (passed in rscratch1)

OopMapSet* Runtime1::generate_exception_throw(StubAssembler* sasm, address target, bool has_argument) {

  // make a frame and preserve the caller's caller-save registers

  OopMap* oop_map = save_live_registers(sasm);

  int call_offset;

  if (!has_argument) {

    call_offset = __ call_RT(noreg, noreg, target);

  } else {

    call_offset = __ call_RT(noreg, noreg, target, rscratch1);

  }

  OopMapSet* oop_maps = new OopMapSet();

  oop_maps->add_gc_map(call_offset, oop_map);

  __ should_not_reach_here();

  return oop_maps;

}

OopMapSet* Runtime1::generate_handle_exception(StubID id, StubAssembler *sasm) {

  __ block_comment("generate_handle_exception");

  // incoming parameters

  const Register exception_oop = r0;

  const Register exception_pc  = r3;

  // other registers used in this stub

  // Save registers, if required.

  OopMapSet* oop_maps = new OopMapSet();

  OopMap* oop_map = NULL;

  switch (id) {

  case forward_exception_id:

    // We're handling an exception in the context of a compiled frame.

    // The registers have been saved in the standard places.  Perform

    // an exception lookup in the caller and dispatch to the handler

    // if found.  Otherwise unwind and dispatch to the callers

    // exception handler.

    oop_map = generate_oop_map(sasm, 1 /*thread*/);

    // load and clear pending exception oop into r0

    __ ldr(exception_oop, Address(rthread, Thread::pending_exception_offset()));

    __ str(zr, Address(rthread, Thread::pending_exception_offset()));

    // load issuing PC (the return address for this stub) into r3

    __ ldr(exception_pc, Address(rfp, 1*BytesPerWord));

    // make sure that the vm_results are cleared (may be unnecessary)

    __ str(zr, Address(rthread, JavaThread::vm_result_offset()));

    __ str(zr, Address(rthread, JavaThread::vm_result_2_offset()));

    break;

  case handle_exception_nofpu_id:

  case handle_exception_id:

    // At this point all registers MAY be live.

    oop_map = save_live_registers(sasm, id != handle_exception_nofpu_id);

    break;

  case handle_exception_from_callee_id: {

    // At this point all registers except exception oop (r0) and

    // exception pc (lr) are dead.

    const int frame_size = 2 /*fp, return address*/;

    oop_map = new OopMap(frame_size * VMRegImpl::slots_per_word, 0);

    sasm->set_frame_size(frame_size);

    break;

  }

  default:

    __ should_not_reach_here();

    break;

  }

  // verify that only r0 and r3 are valid at this time

  __ invalidate_registers(false, true, true, false, true, true);

  // verify that r0 contains a valid exception

  __ verify_not_null_oop(exception_oop);

#ifdef ASSERT

  // check that fields in JavaThread for exception oop and issuing pc are

  // empty before writing to them

  Label oop_empty;

  __ ldr(rscratch1, Address(rthread, JavaThread::exception_oop_offset()));

  __ cbz(rscratch1, oop_empty);

  __ stop("exception oop already set");

  __ bind(oop_empty);

  Label pc_empty;

  __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));

  __ cbz(rscratch1, pc_empty);

  __ stop("exception pc already set");

  __ bind(pc_empty);

#endif

  // save exception oop and issuing pc into JavaThread

  // (exception handler will load it from here)

  __ str(exception_oop, Address(rthread, JavaThread::exception_oop_offset()));

  __ str(exception_pc, Address(rthread, JavaThread::exception_pc_offset()));

  // patch throwing pc into return address (has bci & oop map)

  __ str(exception_pc, Address(rfp, 1*BytesPerWord));

  // compute the exception handler.

  // the exception oop and the throwing pc are read from the fields in JavaThread

  int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, exception_handler_for_pc));

  oop_maps->add_gc_map(call_offset, oop_map);

  // r0: handler address

  //      will be the deopt blob if nmethod was deoptimized while we looked up

  //      handler regardless of whether handler existed in the nmethod.

  // only r0 is valid at this time, all other registers have been destroyed by the runtime call

  __ invalidate_registers(false, true, true, true, true, true);

  // patch the return address, this stub will directly return to the exception handler

  __ str(r0, Address(rfp, 1*BytesPerWord));

  switch (id) {

  case forward_exception_id:

  case handle_exception_nofpu_id:

  case handle_exception_id:

    // Restore the registers that were saved at the beginning.

    restore_live_registers(sasm, id != handle_exception_nofpu_id);

    break;

  case handle_exception_from_callee_id:

    // Pop the return address.

    __ leave();

    __ ret(lr);  // jump to exception handler

    break;

  default:  ShouldNotReachHere();

  }

  return oop_maps;

}

void Runtime1::generate_unwind_exception(StubAssembler *sasm) {

  // incoming parameters

  const Register exception_oop = r0;

  // callee-saved copy of exception_oop during runtime call

  const Register exception_oop_callee_saved = r19;

  // other registers used in this stub

  const Register exception_pc = r3;

  const Register handler_addr = r1;

  // verify that only r0, is valid at this time

  __ invalidate_registers(false, true, true, true, true, true);

#ifdef ASSERT

  // check that fields in JavaThread for exception oop and issuing pc are empty

  Label oop_empty;

  __ ldr(rscratch1, Address(rthread, JavaThread::exception_oop_offset()));

  __ cbz(rscratch1, oop_empty);

  __ stop("exception oop must be empty");

  __ bind(oop_empty);

  Label pc_empty;

  __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));

  __ cbz(rscratch1, pc_empty);

  __ stop("exception pc must be empty");

  __ bind(pc_empty);

#endif

  // Save our return address because

  // exception_handler_for_return_address will destroy it.  We also

  // save exception_oop

  __ stp(lr, exception_oop, Address(__ pre(sp, -2 * wordSize)));

  // search the exception handler address of the caller (using the return address)

  __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, lr);

  // r0: exception handler address of the caller

  // Only R0 is valid at this time; all other registers have been

  // destroyed by the call.

  __ invalidate_registers(false, true, true, true, false, true);

  // move result of call into correct register

  __ mov(handler_addr, r0);

  // get throwing pc (= return address).

  // lr has been destroyed by the call

  __ ldp(lr, exception_oop, Address(__ post(sp, 2 * wordSize)));

  __ mov(r3, lr);

  __ verify_not_null_oop(exception_oop);

  // continue at exception handler (return address removed)

  // note: do *not* remove arguments when unwinding the

  //       activation since the caller assumes having

  //       all arguments on the stack when entering the

  //       runtime to determine the exception handler

  //       (GC happens at call site with arguments!)

  // r0: exception oop

  // r3: throwing pc

  // r1: exception handler

  __ br(handler_addr);

}

OopMapSet* Runtime1::generate_patching(StubAssembler* sasm, address target) {

  // use the maximum number of runtime-arguments here because it is difficult to

  // distinguish each RT-Call.

  // Note: This number affects also the RT-Call in generate_handle_exception because