/*

 * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "asm/macroAssembler.hpp"

#include "asm/macroAssembler.inline.hpp"

#include "code/debugInfoRec.hpp"

#include "code/icBuffer.hpp"

#include "code/vtableStubs.hpp"

#include "interpreter/interpreter.hpp"

#include "oops/compiledICHolder.hpp"

#include "prims/jvmtiRedefineClassesTrace.hpp"

#include "runtime/sharedRuntime.hpp"

#include "runtime/vframeArray.hpp"

#include "vmreg_x86.inline.hpp"

#ifdef COMPILER1

#include "c1/c1_Runtime1.hpp"

#endif

#ifdef COMPILER2

#include "opto/runtime.hpp"

#endif

#define __ masm->

const int StackAlignmentInSlots = StackAlignmentInBytes / VMRegImpl::stack_slot_size;

class SimpleRuntimeFrame {

  public:

  // Most of the runtime stubs have this simple frame layout.

  // This class exists to make the layout shared in one place.

  // Offsets are for compiler stack slots, which are jints.

  enum layout {

    // The frame sender code expects that rbp will be in the "natural" place and

    // will override any oopMap setting for it. We must therefore force the layout

    // so that it agrees with the frame sender code.

    rbp_off = frame::arg_reg_save_area_bytes/BytesPerInt,

    rbp_off2,

    return_off, return_off2,

    framesize

  };

};

class RegisterSaver {

  // Capture info about frame layout.  Layout offsets are in jint

  // units because compiler frame slots are jints.

#define DEF_XMM_OFFS(regnum) xmm ## regnum ## _off = xmm_off + (regnum)*16/BytesPerInt, xmm ## regnum ## H_off

  enum layout {

    fpu_state_off = frame::arg_reg_save_area_bytes/BytesPerInt, // fxsave save area

    xmm_off       = fpu_state_off + 160/BytesPerInt,            // offset in fxsave save area

    DEF_XMM_OFFS(0),

    DEF_XMM_OFFS(1),

    DEF_XMM_OFFS(2),

    DEF_XMM_OFFS(3),

    DEF_XMM_OFFS(4),

    DEF_XMM_OFFS(5),

    DEF_XMM_OFFS(6),

    DEF_XMM_OFFS(7),

    DEF_XMM_OFFS(8),

    DEF_XMM_OFFS(9),

    DEF_XMM_OFFS(10),

    DEF_XMM_OFFS(11),

    DEF_XMM_OFFS(12),

    DEF_XMM_OFFS(13),

    DEF_XMM_OFFS(14),

    DEF_XMM_OFFS(15),

    fpu_state_end = fpu_state_off + ((FPUStateSizeInWords-1)*wordSize / BytesPerInt),

    fpu_stateH_end,

    r15_off, r15H_off,

    r14_off, r14H_off,

    r13_off, r13H_off,

    r12_off, r12H_off,

    r11_off, r11H_off,

    r10_off, r10H_off,

    r9_off,  r9H_off,

    r8_off,  r8H_off,

    rdi_off, rdiH_off,

    rsi_off, rsiH_off,

    ignore_off, ignoreH_off,  // extra copy of rbp

    rsp_off, rspH_off,

    rbx_off, rbxH_off,

    rdx_off, rdxH_off,

    rcx_off, rcxH_off,

    rax_off, raxH_off,

    // 16-byte stack alignment fill word: see MacroAssembler::push/pop_IU_state

    align_off, alignH_off,

    flags_off, flagsH_off,

    // The frame sender code expects that rbp will be in the "natural" place and

    // will override any oopMap setting for it. We must therefore force the layout

    // so that it agrees with the frame sender code.

    rbp_off, rbpH_off,        // copy of rbp we will restore

    return_off, returnH_off,  // slot for return address

    reg_save_size             // size in compiler stack slots

  };

 public:

  static OopMap* save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words, bool save_vectors = false);

  static void restore_live_registers(MacroAssembler* masm, bool restore_vectors = false);

  // Offsets into the register save area

  // Used by deoptimization when it is managing result register

  // values on its own

  static int rax_offset_in_bytes(void)    { return BytesPerInt * rax_off; }

  static int rdx_offset_in_bytes(void)    { return BytesPerInt * rdx_off; }

  static int rbx_offset_in_bytes(void)    { return BytesPerInt * rbx_off; }

  static int xmm0_offset_in_bytes(void)   { return BytesPerInt * xmm0_off; }

  static int return_offset_in_bytes(void) { return BytesPerInt * return_off; }

  // During deoptimization only the result registers need to be restored,

  // all the other values have already been extracted.

  static void restore_result_registers(MacroAssembler* masm);

};

OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words, bool save_vectors) {

  int vect_words = 0;

#ifdef COMPILER2

  if (save_vectors) {

    assert(UseAVX > 0, "256bit vectors are supported only with AVX");

    assert(MaxVectorSize == 32, "only 256bit vectors are supported now");

    // Save upper half of YMM registes

    vect_words = 16 * 16 / wordSize;

    additional_frame_words += vect_words;

  }

#else

  assert(!save_vectors, "vectors are generated only by C2");

#endif

  // Always make the frame size 16-byte aligned

  int frame_size_in_bytes = round_to(additional_frame_words*wordSize +

                                     reg_save_size*BytesPerInt, 16);

  // OopMap frame size is in compiler stack slots (jint's) not bytes or words

  int frame_size_in_slots = frame_size_in_bytes / BytesPerInt;

  // The caller will allocate additional_frame_words

  int additional_frame_slots = additional_frame_words*wordSize / BytesPerInt;

  // CodeBlob frame size is in words.

  int frame_size_in_words = frame_size_in_bytes / wordSize;

  *total_frame_words = frame_size_in_words;

  // Save registers, fpu state, and flags.

  // We assume caller has already pushed the return address onto the

  // stack, so rsp is 8-byte aligned here.

  // We push rpb twice in this sequence because we want the real rbp

  // to be under the return like a normal enter.

  __ enter();          // rsp becomes 16-byte aligned here

  __ push_CPU_state(); // Push a multiple of 16 bytes

  if (vect_words > 0) {

    assert(vect_words*wordSize == 256, "");

    __ subptr(rsp, 256); // Save upper half of YMM registes

    __ vextractf128h(Address(rsp,  0),xmm0);

    __ vextractf128h(Address(rsp, 16),xmm1);

    __ vextractf128h(Address(rsp, 32),xmm2);

    __ vextractf128h(Address(rsp, 48),xmm3);

    __ vextractf128h(Address(rsp, 64),xmm4);

    __ vextractf128h(Address(rsp, 80),xmm5);

    __ vextractf128h(Address(rsp, 96),xmm6);

    __ vextractf128h(Address(rsp,112),xmm7);

    __ vextractf128h(Address(rsp,128),xmm8);

    __ vextractf128h(Address(rsp,144),xmm9);

    __ vextractf128h(Address(rsp,160),xmm10);

    __ vextractf128h(Address(rsp,176),xmm11);

    __ vextractf128h(Address(rsp,192),xmm12);

    __ vextractf128h(Address(rsp,208),xmm13);

    __ vextractf128h(Address(rsp,224),xmm14);

    __ vextractf128h(Address(rsp,240),xmm15);

  }

  if (frame::arg_reg_save_area_bytes != 0) {

    // Allocate argument register save area

    __ subptr(rsp, frame::arg_reg_save_area_bytes);

  }

  // Set an oopmap for the call site.  This oopmap will map all

  // oop-registers and debug-info registers as callee-saved.  This

  // will allow deoptimization at this safepoint to find all possible

  // debug-info recordings, as well as let GC find all oops.

  OopMapSet *oop_maps = new OopMapSet();

  OopMap* map = new OopMap(frame_size_in_slots, 0);

#define STACK_OFFSET(x) VMRegImpl::stack2reg((x) + additional_frame_slots)

  map->set_callee_saved(STACK_OFFSET( rax_off ), rax->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( rcx_off ), rcx->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( rdx_off ), rdx->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( rbx_off ), rbx->as_VMReg());

  // rbp location is known implicitly by the frame sender code, needs no oopmap

  // and the location where rbp was saved by is ignored

  map->set_callee_saved(STACK_OFFSET( rsi_off ), rsi->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( rdi_off ), rdi->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( r8_off  ), r8->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( r9_off  ), r9->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( r10_off ), r10->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( r11_off ), r11->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( r12_off ), r12->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( r13_off ), r13->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( r14_off ), r14->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( r15_off ), r15->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm0_off ), xmm0->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm1_off ), xmm1->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm2_off ), xmm2->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm3_off ), xmm3->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm4_off ), xmm4->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm5_off ), xmm5->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm6_off ), xmm6->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm7_off ), xmm7->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm8_off ), xmm8->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm9_off ), xmm9->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm10_off), xmm10->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm11_off), xmm11->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm12_off), xmm12->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm13_off), xmm13->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm14_off), xmm14->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm15_off), xmm15->as_VMReg());

  // %%% These should all be a waste but we'll keep things as they were for now

  if (true) {

    map->set_callee_saved(STACK_OFFSET( raxH_off ), rax->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET( rcxH_off ), rcx->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET( rdxH_off ), rdx->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET( rbxH_off ), rbx->as_VMReg()->next());

    // rbp location is known implicitly by the frame sender code, needs no oopmap

    map->set_callee_saved(STACK_OFFSET( rsiH_off ), rsi->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET( rdiH_off ), rdi->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET( r8H_off  ), r8->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET( r9H_off  ), r9->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET( r10H_off ), r10->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET( r11H_off ), r11->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET( r12H_off ), r12->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET( r13H_off ), r13->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET( r14H_off ), r14->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET( r15H_off ), r15->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET(xmm0H_off ), xmm0->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET(xmm1H_off ), xmm1->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET(xmm2H_off ), xmm2->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET(xmm3H_off ), xmm3->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET(xmm4H_off ), xmm4->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET(xmm5H_off ), xmm5->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET(xmm6H_off ), xmm6->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET(xmm7H_off ), xmm7->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET(xmm8H_off ), xmm8->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET(xmm9H_off ), xmm9->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET(xmm10H_off), xmm10->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET(xmm11H_off), xmm11->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET(xmm12H_off), xmm12->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET(xmm13H_off), xmm13->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET(xmm14H_off), xmm14->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET(xmm15H_off), xmm15->as_VMReg()->next());

  }

  return map;

}

void RegisterSaver::restore_live_registers(MacroAssembler* masm, bool restore_vectors) {

  if (frame::arg_reg_save_area_bytes != 0) {

    // Pop arg register save area

    __ addptr(rsp, frame::arg_reg_save_area_bytes);

  }

#ifdef COMPILER2

  if (restore_vectors) {

    // Restore upper half of YMM registes.

    assert(UseAVX > 0, "256bit vectors are supported only with AVX");

    assert(MaxVectorSize == 32, "only 256bit vectors are supported now");

    __ vinsertf128h(xmm0, Address(rsp,  0));

    __ vinsertf128h(xmm1, Address(rsp, 16));

    __ vinsertf128h(xmm2, Address(rsp, 32));

    __ vinsertf128h(xmm3, Address(rsp, 48));

    __ vinsertf128h(xmm4, Address(rsp, 64));

    __ vinsertf128h(xmm5, Address(rsp, 80));

    __ vinsertf128h(xmm6, Address(rsp, 96));

    __ vinsertf128h(xmm7, Address(rsp,112));

    __ vinsertf128h(xmm8, Address(rsp,128));

    __ vinsertf128h(xmm9, Address(rsp,144));

    __ vinsertf128h(xmm10, Address(rsp,160));

    __ vinsertf128h(xmm11, Address(rsp,176));

    __ vinsertf128h(xmm12, Address(rsp,192));

    __ vinsertf128h(xmm13, Address(rsp,208));

    __ vinsertf128h(xmm14, Address(rsp,224));

    __ vinsertf128h(xmm15, Address(rsp,240));

    __ addptr(rsp, 256);

  }

#else

  assert(!restore_vectors, "vectors are generated only by C2");

#endif

  // Recover CPU state

  __ pop_CPU_state();

  // Get the rbp described implicitly by the calling convention (no oopMap)

  __ pop(rbp);

}

void RegisterSaver::restore_result_registers(MacroAssembler* masm) {

  // Just restore result register. Only used by deoptimization. By

  // now any callee save register that needs to be restored to a c2

  // caller of the deoptee has been extracted into the vframeArray

  // and will be stuffed into the c2i adapter we create for later

  // restoration so only result registers need to be restored here.

  // Restore fp result register

  __ movdbl(xmm0, Address(rsp, xmm0_offset_in_bytes()));

  // Restore integer result register

  __ movptr(rax, Address(rsp, rax_offset_in_bytes()));

  __ movptr(rdx, Address(rsp, rdx_offset_in_bytes()));

  // Pop all of the register save are off the stack except the return address

  __ addptr(rsp, return_offset_in_bytes());

}

// Is vector's size (in bytes) bigger than a size saved by default?

// 16 bytes XMM registers are saved by default using fxsave/fxrstor instructions.

bool SharedRuntime::is_wide_vector(int size) {

  return size > 16;

}

// The java_calling_convention describes stack locations as ideal slots on

// a frame with no abi restrictions. Since we must observe abi restrictions

// (like the placement of the register window) the slots must be biased by

// the following value.

static int reg2offset_in(VMReg r) {

  // Account for saved rbp and return address

  // This should really be in_preserve_stack_slots

  return (r->reg2stack() + 4) * VMRegImpl::stack_slot_size;

}

static int reg2offset_out(VMReg r) {

  return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;

}

// ---------------------------------------------------------------------------

// Read the array of BasicTypes from a signature, and compute where the

// arguments should go.  Values in the VMRegPair regs array refer to 4-byte

// quantities.  Values less than VMRegImpl::stack0 are registers, those above

// refer to 4-byte stack slots.  All stack slots are based off of the stack pointer

// as framesizes are fixed.

// VMRegImpl::stack0 refers to the first slot 0(sp).

// and VMRegImpl::stack0+1 refers to the memory word 4-byes higher.  Register

// up to RegisterImpl::number_of_registers) are the 64-bit

// integer registers.

// Note: the INPUTS in sig_bt are in units of Java argument words, which are

// either 32-bit or 64-bit depending on the build.  The OUTPUTS are in 32-bit

// units regardless of build. Of course for i486 there is no 64 bit build

// The Java calling convention is a "shifted" version of the C ABI.

// By skipping the first C ABI register we can call non-static jni methods

// with small numbers of arguments without having to shuffle the arguments

// at all. Since we control the java ABI we ought to at least get some

// advantage out of it.

int SharedRuntime::java_calling_convention(const BasicType *sig_bt,

                                           VMRegPair *regs,

                                           int total_args_passed,

                                           int is_outgoing) {

  // Create the mapping between argument positions and

  // registers.

  static const Register INT_ArgReg[Argument::n_int_register_parameters_j] = {

    j_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4, j_rarg5

  };

  static const XMMRegister FP_ArgReg[Argument::n_float_register_parameters_j] = {

    j_farg0, j_farg1, j_farg2, j_farg3,

    j_farg4, j_farg5, j_farg6, j_farg7

  };

  uint int_args = 0;

  uint fp_args = 0;

  uint stk_args = 0; // inc by 2 each time

  for (int i = 0; i < total_args_passed; i++) {

    switch (sig_bt[i]) {

    case T_BOOLEAN:

    case T_CHAR:

    case T_BYTE:

    case T_SHORT:

    case T_INT:

      if (int_args < Argument::n_int_register_parameters_j) {

        regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());

      } else {

        regs[i].set1(VMRegImpl::stack2reg(stk_args));

        stk_args += 2;

      }

      break;

    case T_VOID:

      // halves of T_LONG or T_DOUBLE

      assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");

      regs[i].set_bad();

      break;

    case T_LONG:

      assert(sig_bt[i + 1] == T_VOID, "expecting half");

      // fall through

    case T_OBJECT:

    case T_ARRAY:

    case T_ADDRESS:

      if (int_args < Argument::n_int_register_parameters_j) {

        regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());

      } else {

        regs[i].set2(VMRegImpl::stack2reg(stk_args));

        stk_args += 2;

      }

      break;

    case T_FLOAT:

      if (fp_args < Argument::n_float_register_parameters_j) {

        regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());

      } else {

        regs[i].set1(VMRegImpl::stack2reg(stk_args));

        stk_args += 2;

      }

      break;

    case T_DOUBLE:

      assert(sig_bt[i + 1] == T_VOID, "expecting half");

      if (fp_args < Argument::n_float_register_parameters_j) {

        regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg());

      } else {

        regs[i].set2(VMRegImpl::stack2reg(stk_args));

        stk_args += 2;

      }

      break;

    default:

      ShouldNotReachHere();

      break;

    }

  }

  return round_to(stk_args, 2);

}

// Patch the callers callsite with entry to compiled code if it exists.

static void patch_callers_callsite(MacroAssembler *masm) {

  Label L;

  __ cmpptr(Address(rbx, in_bytes(Method::code_offset())), (int32_t)NULL_WORD);

  __ jcc(Assembler::equal, L);

  // Save the current stack pointer

  __ mov(r13, rsp);

  // Schedule the branch target address early.

  // Call into the VM to patch the caller, then jump to compiled callee

  // rax isn't live so capture return address while we easily can

  __ movptr(rax, Address(rsp, 0));

  // align stack so push_CPU_state doesn't fault

  __ andptr(rsp, -(StackAlignmentInBytes));

  __ push_CPU_state();

  // VM needs caller's callsite

  // VM needs target method

  // This needs to be a long call since we will relocate this adapter to

  // the codeBuffer and it may not reach

  // Allocate argument register save area

  if (frame::arg_reg_save_area_bytes != 0) {

    __ subptr(rsp, frame::arg_reg_save_area_bytes);

  }

  __ mov(c_rarg0, rbx);

  __ mov(c_rarg1, rax);

  __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite)));

  // De-allocate argument register save area

  if (frame::arg_reg_save_area_bytes != 0) {

    __ addptr(rsp, frame::arg_reg_save_area_bytes);

  }

  __ pop_CPU_state();

  // restore sp

  __ mov(rsp, r13);

  __ bind(L);

}

static void gen_c2i_adapter(MacroAssembler *masm,

                            int total_args_passed,

                            int comp_args_on_stack,

                            const BasicType *sig_bt,

                            const VMRegPair *regs,

                            Label& skip_fixup) {

  // Before we get into the guts of the C2I adapter, see if we should be here

  // at all.  We've come from compiled code and are attempting to jump to the

  // interpreter, which means the caller made a static call to get here

  // (vcalls always get a compiled target if there is one).  Check for a

  // compiled target.  If there is one, we need to patch the caller's call.

  patch_callers_callsite(masm);

  __ bind(skip_fixup);

  // Since all args are passed on the stack, total_args_passed *

  // Interpreter::stackElementSize is the space we need. Plus 1 because

  // we also account for the return address location since

  // we store it first rather than hold it in rax across all the shuffling