/*

 * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.

 * Copyright (c) 2014, 2019, Red Hat Inc. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "asm/macroAssembler.hpp"

#include "asm/macroAssembler.inline.hpp"

#include "code/debugInfoRec.hpp"

#include "code/icBuffer.hpp"

#include "code/vtableStubs.hpp"

#include "interpreter/interpreter.hpp"

#include "interpreter/interp_masm.hpp"

#include "logging/log.hpp"

#include "memory/resourceArea.hpp"

#include "oops/compiledICHolder.hpp"

#include "runtime/safepointMechanism.hpp"

#include "runtime/sharedRuntime.hpp"

#include "runtime/vframeArray.hpp"

#include "utilities/align.hpp"

#include "vmreg_aarch64.inline.hpp"

#ifdef COMPILER1

#include "c1/c1_Runtime1.hpp"

#endif

#if COMPILER2_OR_JVMCI

#include "adfiles/ad_aarch64.hpp"

#include "opto/runtime.hpp"

#endif

#if INCLUDE_JVMCI

#include "jvmci/jvmciJavaClasses.hpp"

#endif

#define __ masm->

const int StackAlignmentInSlots = StackAlignmentInBytes / VMRegImpl::stack_slot_size;

class SimpleRuntimeFrame {

  public:

  // Most of the runtime stubs have this simple frame layout.

  // This class exists to make the layout shared in one place.

  // Offsets are for compiler stack slots, which are jints.

  enum layout {

    // The frame sender code expects that rbp will be in the "natural" place and

    // will override any oopMap setting for it. We must therefore force the layout

    // so that it agrees with the frame sender code.

    // we don't expect any arg reg save area so aarch64 asserts that

    // frame::arg_reg_save_area_bytes == 0

    rbp_off = 0,

    rbp_off2,

    return_off, return_off2,

    framesize

  };

};

// FIXME -- this is used by C1

class RegisterSaver {

 public:

  static OopMap* save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words, bool save_vectors = false);

  static void restore_live_registers(MacroAssembler* masm, bool restore_vectors = false);

  // Offsets into the register save area

  // Used by deoptimization when it is managing result register

  // values on its own

  static int r0_offset_in_bytes(void)    { return (32 + r0->encoding()) * wordSize; }

  static int reg_offset_in_bytes(Register r)    { return r0_offset_in_bytes() + r->encoding() * wordSize; }

  static int rmethod_offset_in_bytes(void)    { return reg_offset_in_bytes(rmethod); }

  static int rscratch1_offset_in_bytes(void)    { return (32 + rscratch1->encoding()) * wordSize; }

  static int v0_offset_in_bytes(void)   { return 0; }

  static int return_offset_in_bytes(void) { return (32 /* floats*/ + 31 /* gregs*/) * wordSize; }

  // During deoptimization only the result registers need to be restored,

  // all the other values have already been extracted.

  static void restore_result_registers(MacroAssembler* masm);

    // Capture info about frame layout

  enum layout {

                fpu_state_off = 0,

                fpu_state_end = fpu_state_off+FPUStateSizeInWords-1,

                // The frame sender code expects that rfp will be in

                // the "natural" place and will override any oopMap

                // setting for it. We must therefore force the layout

                // so that it agrees with the frame sender code.

                r0_off = fpu_state_off+FPUStateSizeInWords,

                rfp_off = r0_off + 30 * 2,

                return_off = rfp_off + 2,      // slot for return address

                reg_save_size = return_off + 2};

};

OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words, bool save_vectors) {

#if COMPILER2_OR_JVMCI

  if (save_vectors) {

    // Save upper half of vector registers

    int vect_words = 32 * 8 / wordSize;

    additional_frame_words += vect_words;

  }

#else

  assert(!save_vectors, "vectors are generated only by C2 and JVMCI");

#endif

  int frame_size_in_bytes = align_up(additional_frame_words*wordSize +

                                     reg_save_size*BytesPerInt, 16);

  // OopMap frame size is in compiler stack slots (jint's) not bytes or words

  int frame_size_in_slots = frame_size_in_bytes / BytesPerInt;

  // The caller will allocate additional_frame_words

  int additional_frame_slots = additional_frame_words*wordSize / BytesPerInt;

  // CodeBlob frame size is in words.

  int frame_size_in_words = frame_size_in_bytes / wordSize;

  *total_frame_words = frame_size_in_words;

  // Save Integer and Float registers.

  __ enter();

  __ push_CPU_state(save_vectors);

  // Set an oopmap for the call site.  This oopmap will map all

  // oop-registers and debug-info registers as callee-saved.  This

  // will allow deoptimization at this safepoint to find all possible

  // debug-info recordings, as well as let GC find all oops.

  OopMapSet *oop_maps = new OopMapSet();

  OopMap* oop_map = new OopMap(frame_size_in_slots, 0);

  for (int i = 0; i < RegisterImpl::number_of_registers; i++) {

    Register r = as_Register(i);

    if (r < rheapbase && r != rscratch1 && r != rscratch2) {

      int sp_offset = 2 * (i + 32); // SP offsets are in 4-byte words,

                                    // register slots are 8 bytes

                                    // wide, 32 floating-point

                                    // registers

      oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset + additional_frame_slots),

                                r->as_VMReg());

    }

  }

  for (int i = 0; i < FloatRegisterImpl::number_of_registers; i++) {

    FloatRegister r = as_FloatRegister(i);

    int sp_offset = save_vectors ? (4 * i) : (2 * i);

    oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset),

                              r->as_VMReg());

  }

  return oop_map;

}

void RegisterSaver::restore_live_registers(MacroAssembler* masm, bool restore_vectors) {

#ifndef COMPILER2

  assert(!restore_vectors, "vectors are generated only by C2 and JVMCI");

#endif

  __ pop_CPU_state(restore_vectors);

  __ leave();

}

void RegisterSaver::restore_result_registers(MacroAssembler* masm) {

  // Just restore result register. Only used by deoptimization. By

  // now any callee save register that needs to be restored to a c2

  // caller of the deoptee has been extracted into the vframeArray

  // and will be stuffed into the c2i adapter we create for later

  // restoration so only result registers need to be restored here.

  // Restore fp result register

  __ ldrd(v0, Address(sp, v0_offset_in_bytes()));

  // Restore integer result register

  __ ldr(r0, Address(sp, r0_offset_in_bytes()));

  // Pop all of the register save are off the stack

  __ add(sp, sp, align_up(return_offset_in_bytes(), 16));

}

// Is vector's size (in bytes) bigger than a size saved by default?

// 8 bytes vector registers are saved by default on AArch64.

bool SharedRuntime::is_wide_vector(int size) {

  return size > 8;

}

size_t SharedRuntime::trampoline_size() {

  return 16;

}

void SharedRuntime::generate_trampoline(MacroAssembler *masm, address destination) {

  __ mov(rscratch1, destination);

  __ br(rscratch1);

}

// The java_calling_convention describes stack locations as ideal slots on

// a frame with no abi restrictions. Since we must observe abi restrictions

// (like the placement of the register window) the slots must be biased by

// the following value.

static int reg2offset_in(VMReg r) {

  // Account for saved rfp and lr

  // This should really be in_preserve_stack_slots

  return (r->reg2stack() + 4) * VMRegImpl::stack_slot_size;

}

static int reg2offset_out(VMReg r) {

  return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;

}

// ---------------------------------------------------------------------------

// Read the array of BasicTypes from a signature, and compute where the

// arguments should go.  Values in the VMRegPair regs array refer to 4-byte

// quantities.  Values less than VMRegImpl::stack0 are registers, those above

// refer to 4-byte stack slots.  All stack slots are based off of the stack pointer

// as framesizes are fixed.

// VMRegImpl::stack0 refers to the first slot 0(sp).

// and VMRegImpl::stack0+1 refers to the memory word 4-byes higher.  Register

// up to RegisterImpl::number_of_registers) are the 64-bit

// integer registers.

// Note: the INPUTS in sig_bt are in units of Java argument words,

// which are 64-bit.  The OUTPUTS are in 32-bit units.

// The Java calling convention is a "shifted" version of the C ABI.

// By skipping the first C ABI register we can call non-static jni

// methods with small numbers of arguments without having to shuffle

// the arguments at all. Since we control the java ABI we ought to at

// least get some advantage out of it.

int SharedRuntime::java_calling_convention(const BasicType *sig_bt,

                                           VMRegPair *regs,

                                           int total_args_passed,

                                           int is_outgoing) {

  // Create the mapping between argument positions and

  // registers.

  static const Register INT_ArgReg[Argument::n_int_register_parameters_j] = {

    j_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4, j_rarg5, j_rarg6, j_rarg7

  };

  static const FloatRegister FP_ArgReg[Argument::n_float_register_parameters_j] = {

    j_farg0, j_farg1, j_farg2, j_farg3,

    j_farg4, j_farg5, j_farg6, j_farg7

  };

  uint int_args = 0;

  uint fp_args = 0;

  uint stk_args = 0; // inc by 2 each time

  for (int i = 0; i < total_args_passed; i++) {

    switch (sig_bt[i]) {

    case T_BOOLEAN:

    case T_CHAR:

    case T_BYTE:

    case T_SHORT:

    case T_INT:

      if (int_args < Argument::n_int_register_parameters_j) {

        regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());

      } else {

        regs[i].set1(VMRegImpl::stack2reg(stk_args));

        stk_args += 2;

      }

      break;

    case T_VOID:

      // halves of T_LONG or T_DOUBLE

      assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");

      regs[i].set_bad();

      break;

    case T_LONG:

      assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");

      // fall through

    case T_OBJECT:

    case T_ARRAY:

    case T_ADDRESS:

      if (int_args < Argument::n_int_register_parameters_j) {

        regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());

      } else {

        regs[i].set2(VMRegImpl::stack2reg(stk_args));

        stk_args += 2;

      }

      break;

    case T_FLOAT:

      if (fp_args < Argument::n_float_register_parameters_j) {

        regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());

      } else {

        regs[i].set1(VMRegImpl::stack2reg(stk_args));

        stk_args += 2;

      }

      break;

    case T_DOUBLE:

      assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");

      if (fp_args < Argument::n_float_register_parameters_j) {

        regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg());

      } else {

        regs[i].set2(VMRegImpl::stack2reg(stk_args));

        stk_args += 2;

      }

      break;

    default:

      ShouldNotReachHere();

      break;

    }

  }

  return align_up(stk_args, 2);

}

// Patch the callers callsite with entry to compiled code if it exists.

static void patch_callers_callsite(MacroAssembler *masm) {

  Label L;

  __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));

  __ cbz(rscratch1, L);

  __ enter();

  __ push_CPU_state();

  // VM needs caller's callsite

  // VM needs target method

  // This needs to be a long call since we will relocate this adapter to

  // the codeBuffer and it may not reach

#ifndef PRODUCT

  assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");

#endif

  __ mov(c_rarg0, rmethod);

  __ mov(c_rarg1, lr);

  __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite)));

  __ maybe_isb();

  __ pop_CPU_state();

  // restore sp

  __ leave();

  __ bind(L);

}

static void gen_c2i_adapter(MacroAssembler *masm,

                            int total_args_passed,

                            int comp_args_on_stack,

                            const BasicType *sig_bt,

                            const VMRegPair *regs,

                            Label& skip_fixup) {

  // Before we get into the guts of the C2I adapter, see if we should be here

  // at all.  We've come from compiled code and are attempting to jump to the

  // interpreter, which means the caller made a static call to get here

  // (vcalls always get a compiled target if there is one).  Check for a

  // compiled target.  If there is one, we need to patch the caller's call.

  patch_callers_callsite(masm);

  __ bind(skip_fixup);

  int words_pushed = 0;

  // Since all args are passed on the stack, total_args_passed *

  // Interpreter::stackElementSize is the space we need.

  int extraspace = total_args_passed * Interpreter::stackElementSize;

  __ mov(r13, sp);

  // stack is aligned, keep it that way

  extraspace = align_up(extraspace, 2*wordSize);

  if (extraspace)

    __ sub(sp, sp, extraspace);

  // Now write the args into the outgoing interpreter space

  for (int i = 0; i < total_args_passed; i++) {

    if (sig_bt[i] == T_VOID) {

      assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");

      continue;

    }

    // offset to start parameters

    int st_off   = (total_args_passed - i - 1) * Interpreter::stackElementSize;

    int next_off = st_off - Interpreter::stackElementSize;

    // Say 4 args:

    // i   st_off

    // 0   32 T_LONG

    // 1   24 T_VOID

    // 2   16 T_OBJECT

    // 3    8 T_BOOL

    // -    0 return address

    //

    // However to make thing extra confusing. Because we can fit a long/double in

    // a single slot on a 64 bt vm and it would be silly to break them up, the interpreter

    // leaves one slot empty and only stores to a single slot. In this case the

    // slot that is occupied is the T_VOID slot. See I said it was confusing.

    VMReg r_1 = regs[i].first();

    VMReg r_2 = regs[i].second();

    if (!r_1->is_valid()) {

      assert(!r_2->is_valid(), "");

      continue;

    }

    if (r_1->is_stack()) {

      // memory to memory use rscratch1

      int ld_off = (r_1->reg2stack() * VMRegImpl::stack_slot_size

                    + extraspace

                    + words_pushed * wordSize);

      if (!r_2->is_valid()) {

        // sign extend??

        __ ldrw(rscratch1, Address(sp, ld_off));

        __ str(rscratch1, Address(sp, st_off));

      } else {

        __ ldr(rscratch1, Address(sp, ld_off));

        // Two VMREgs|OptoRegs can be T_OBJECT, T_ADDRESS, T_DOUBLE, T_LONG

        // T_DOUBLE and T_LONG use two slots in the interpreter

        if ( sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) {

          // ld_off == LSW, ld_off+wordSize == MSW

          // st_off == MSW, next_off == LSW

          __ str(rscratch1, Address(sp, next_off));

#ifdef ASSERT

          // Overwrite the unused slot with known junk

          __ mov(rscratch1, 0xdeadffffdeadaaaaul);

          __ str(rscratch1, Address(sp, st_off));

#endif /* ASSERT */

        } else {

          __ str(rscratch1, Address(sp, st_off));

        }

      }

    } else if (r_1->is_Register()) {

      Register r = r_1->as_Register();

      if (!r_2->is_valid()) {

        // must be only an int (or less ) so move only 32bits to slot

        // why not sign extend??

        __ str(r, Address(sp, st_off));

      } else {

        // Two VMREgs|OptoRegs can be T_OBJECT, T_ADDRESS, T_DOUBLE, T_LONG

        // T_DOUBLE and T_LONG use two slots in the interpreter

        if ( sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) {

          // long/double in gpr

#ifdef ASSERT

          // Overwrite the unused slot with known junk

          __ mov(rscratch1, 0xdeadffffdeadaaabul);

          __ str(rscratch1, Address(sp, st_off));

#endif /* ASSERT */

          __ str(r, Address(sp, next_off));

        } else {

          __ str(r, Address(sp, st_off));

        }

      }

    } else {

      assert(r_1->is_FloatRegister(), "");

      if (!r_2->is_valid()) {

        // only a float use just part of the slot

        __ strs(r_1->as_FloatRegister(), Address(sp, st_off));

      } else {

#ifdef ASSERT

        // Overwrite the unused slot with known junk

        __ mov(rscratch1, 0xdeadffffdeadaaacul);

        __ str(rscratch1, Address(sp, st_off));

#endif /* ASSERT */

        __ strd(r_1->as_FloatRegister(), Address(sp, next_off));

      }

    }

  }

  __ mov(esp, sp); // Interp expects args on caller's expression stack

  __ ldr(rscratch1, Address(rmethod, in_bytes(Method::interpreter_entry_offset())));

  __ br(rscratch1);

}

void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm,

                                    int total_args_passed,

                                    int comp_args_on_stack,

                                    const BasicType *sig_bt,

                                    const VMRegPair *regs) {

  // Note: r13 contains the senderSP on entry. We must preserve it since

  // we may do a i2c -> c2i transition if we lose a race where compiled

  // code goes non-entrant while we get args ready.

  // In addition we use r13 to locate all the interpreter args because

  // we must align the stack to 16 bytes.

  // Adapters are frameless.

  // An i2c adapter is frameless because the *caller* frame, which is

  // interpreted, routinely repairs its own esp (from

  // interpreter_frame_last_sp), even if a callee has modified the

  // stack pointer.  It also recalculates and aligns sp.

  // A c2i adapter is frameless because the *callee* frame, which is

  // interpreted, routinely repairs its caller's sp (from sender_sp,

  // which is set up via the senderSP register).

  // In other words, if *either* the caller or callee is interpreted, we can

  // get the stack pointer repaired after a call.

  // This is why c2i and i2c adapters cannot be indefinitely composed.

  // In particular, if a c2i adapter were to somehow call an i2c adapter,

  // both caller and callee would be compiled methods, and neither would

  // clean up the stack pointer changes performed by the two adapters.

  // If this happens, control eventually transfers back to the compiled

  // caller, but with an uncorrected stack, causing delayed havoc.

  if (VerifyAdapterCalls &&

      (Interpreter::code() != NULL || StubRoutines::code1() != NULL)) {

#if 0

    // So, let's test for cascading c2i/i2c adapters right now.

    //  assert(Interpreter::contains($return_addr) ||