/*

 * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "asm/macroAssembler.hpp"

#include "asm/macroAssembler.inline.hpp"

#include "code/debugInfoRec.hpp"

#include "code/icBuffer.hpp"

#include "code/vtableStubs.hpp"

#include "interpreter/interpreter.hpp"

#include "oops/compiledICHolder.hpp"

#include "prims/jvmtiRedefineClassesTrace.hpp"

#include "runtime/sharedRuntime.hpp"

#include "runtime/vframeArray.hpp"

#include "vmreg_x86.inline.hpp"

#ifdef COMPILER1

#include "c1/c1_Runtime1.hpp"

#endif

#ifdef COMPILER2

#include "opto/runtime.hpp"

#endif

#define __ masm->

const int StackAlignmentInSlots = StackAlignmentInBytes / VMRegImpl::stack_slot_size;

class RegisterSaver {

  // Capture info about frame layout

#define DEF_XMM_OFFS(regnum) xmm ## regnum ## _off = xmm_off + (regnum)*16/BytesPerInt, xmm ## regnum ## H_off

  enum layout {

                fpu_state_off = 0,

                fpu_state_end = fpu_state_off+FPUStateSizeInWords,

                st0_off, st0H_off,

                st1_off, st1H_off,

                st2_off, st2H_off,

                st3_off, st3H_off,

                st4_off, st4H_off,

                st5_off, st5H_off,

                st6_off, st6H_off,

                st7_off, st7H_off,

                xmm_off,

                DEF_XMM_OFFS(0),

                DEF_XMM_OFFS(1),

                DEF_XMM_OFFS(2),

                DEF_XMM_OFFS(3),

                DEF_XMM_OFFS(4),

                DEF_XMM_OFFS(5),

                DEF_XMM_OFFS(6),

                DEF_XMM_OFFS(7),

                flags_off = xmm7_off + 16/BytesPerInt + 1, // 16-byte stack alignment fill word

                rdi_off,

                rsi_off,

                ignore_off,  // extra copy of rbp,

                rsp_off,

                rbx_off,

                rdx_off,

                rcx_off,

                rax_off,

                // The frame sender code expects that rbp will be in the "natural" place and

                // will override any oopMap setting for it. We must therefore force the layout

                // so that it agrees with the frame sender code.

                rbp_off,

                return_off,      // slot for return address

                reg_save_size };

  enum { FPU_regs_live = flags_off - fpu_state_end };

  public:

  static OopMap* save_live_registers(MacroAssembler* masm, int additional_frame_words,

                                     int* total_frame_words, bool verify_fpu = true, bool save_vectors = false);

  static void restore_live_registers(MacroAssembler* masm, bool restore_vectors = false);

  static int rax_offset() { return rax_off; }

  static int rbx_offset() { return rbx_off; }

  // Offsets into the register save area

  // Used by deoptimization when it is managing result register

  // values on its own

  static int raxOffset(void) { return rax_off; }

  static int rdxOffset(void) { return rdx_off; }

  static int rbxOffset(void) { return rbx_off; }

  static int xmm0Offset(void) { return xmm0_off; }

  // This really returns a slot in the fp save area, which one is not important

  static int fpResultOffset(void) { return st0_off; }

  // During deoptimization only the result register need to be restored

  // all the other values have already been extracted.

  static void restore_result_registers(MacroAssembler* masm);

};

OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, int additional_frame_words,

                                           int* total_frame_words, bool verify_fpu, bool save_vectors) {

  int vect_words = 0;

#ifdef COMPILER2

  if (save_vectors) {

    assert(UseAVX > 0, "256bit vectors are supported only with AVX");

    assert(MaxVectorSize == 32, "only 256bit vectors are supported now");

    // Save upper half of YMM registes

    vect_words = 8 * 16 / wordSize;

    additional_frame_words += vect_words;

  }

#else

  assert(!save_vectors, "vectors are generated only by C2");

#endif

  int frame_size_in_bytes = (reg_save_size + additional_frame_words) * wordSize;

  int frame_words = frame_size_in_bytes / wordSize;

  *total_frame_words = frame_words;

  assert(FPUStateSizeInWords == 27, "update stack layout");

  // save registers, fpu state, and flags

  // We assume caller has already has return address slot on the stack

  // We push epb twice in this sequence because we want the real rbp,

  // to be under the return like a normal enter and we want to use pusha

  // We push by hand instead of pusing push

  __ enter();

  __ pusha();

  __ pushf();

  __ subptr(rsp,FPU_regs_live*wordSize); // Push FPU registers space

  __ push_FPU_state();          // Save FPU state & init

  if (verify_fpu) {

    // Some stubs may have non standard FPU control word settings so

    // only check and reset the value when it required to be the

    // standard value.  The safepoint blob in particular can be used

    // in methods which are using the 24 bit control word for

    // optimized float math.

#ifdef ASSERT

    // Make sure the control word has the expected value

    Label ok;

    __ cmpw(Address(rsp, 0), StubRoutines::fpu_cntrl_wrd_std());

    __ jccb(Assembler::equal, ok);

    __ stop("corrupted control word detected");

    __ bind(ok);

#endif

    // Reset the control word to guard against exceptions being unmasked

    // since fstp_d can cause FPU stack underflow exceptions.  Write it

    // into the on stack copy and then reload that to make sure that the

    // current and future values are correct.

    __ movw(Address(rsp, 0), StubRoutines::fpu_cntrl_wrd_std());

  }

  __ frstor(Address(rsp, 0));

  if (!verify_fpu) {

    // Set the control word so that exceptions are masked for the

    // following code.

    __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));

  }

  // Save the FPU registers in de-opt-able form

  __ fstp_d(Address(rsp, st0_off*wordSize)); // st(0)

  __ fstp_d(Address(rsp, st1_off*wordSize)); // st(1)

  __ fstp_d(Address(rsp, st2_off*wordSize)); // st(2)

  __ fstp_d(Address(rsp, st3_off*wordSize)); // st(3)

  __ fstp_d(Address(rsp, st4_off*wordSize)); // st(4)

  __ fstp_d(Address(rsp, st5_off*wordSize)); // st(5)

  __ fstp_d(Address(rsp, st6_off*wordSize)); // st(6)

  __ fstp_d(Address(rsp, st7_off*wordSize)); // st(7)

  if( UseSSE == 1 ) {           // Save the XMM state

    __ movflt(Address(rsp,xmm0_off*wordSize),xmm0);

    __ movflt(Address(rsp,xmm1_off*wordSize),xmm1);

    __ movflt(Address(rsp,xmm2_off*wordSize),xmm2);

    __ movflt(Address(rsp,xmm3_off*wordSize),xmm3);

    __ movflt(Address(rsp,xmm4_off*wordSize),xmm4);

    __ movflt(Address(rsp,xmm5_off*wordSize),xmm5);

    __ movflt(Address(rsp,xmm6_off*wordSize),xmm6);

    __ movflt(Address(rsp,xmm7_off*wordSize),xmm7);

  } else if( UseSSE >= 2 ) {

    // Save whole 128bit (16 bytes) XMM regiters

    __ movdqu(Address(rsp,xmm0_off*wordSize),xmm0);

    __ movdqu(Address(rsp,xmm1_off*wordSize),xmm1);

    __ movdqu(Address(rsp,xmm2_off*wordSize),xmm2);

    __ movdqu(Address(rsp,xmm3_off*wordSize),xmm3);

    __ movdqu(Address(rsp,xmm4_off*wordSize),xmm4);

    __ movdqu(Address(rsp,xmm5_off*wordSize),xmm5);

    __ movdqu(Address(rsp,xmm6_off*wordSize),xmm6);

    __ movdqu(Address(rsp,xmm7_off*wordSize),xmm7);

  }

  if (vect_words > 0) {

    assert(vect_words*wordSize == 128, "");

    __ subptr(rsp, 128); // Save upper half of YMM registes

    __ vextractf128h(Address(rsp,  0),xmm0);

    __ vextractf128h(Address(rsp, 16),xmm1);

    __ vextractf128h(Address(rsp, 32),xmm2);

    __ vextractf128h(Address(rsp, 48),xmm3);

    __ vextractf128h(Address(rsp, 64),xmm4);

    __ vextractf128h(Address(rsp, 80),xmm5);

    __ vextractf128h(Address(rsp, 96),xmm6);

    __ vextractf128h(Address(rsp,112),xmm7);

  }

  // Set an oopmap for the call site.  This oopmap will map all

  // oop-registers and debug-info registers as callee-saved.  This

  // will allow deoptimization at this safepoint to find all possible

  // debug-info recordings, as well as let GC find all oops.

  OopMapSet *oop_maps = new OopMapSet();

  OopMap* map =  new OopMap( frame_words, 0 );

#define STACK_OFFSET(x) VMRegImpl::stack2reg((x) + additional_frame_words)

  map->set_callee_saved(STACK_OFFSET( rax_off), rax->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( rcx_off), rcx->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( rdx_off), rdx->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( rbx_off), rbx->as_VMReg());

  // rbp, location is known implicitly, no oopMap

  map->set_callee_saved(STACK_OFFSET( rsi_off), rsi->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( rdi_off), rdi->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(st0_off), as_FloatRegister(0)->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(st1_off), as_FloatRegister(1)->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(st2_off), as_FloatRegister(2)->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(st3_off), as_FloatRegister(3)->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(st4_off), as_FloatRegister(4)->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(st5_off), as_FloatRegister(5)->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(st6_off), as_FloatRegister(6)->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(st7_off), as_FloatRegister(7)->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm0_off), xmm0->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm1_off), xmm1->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm2_off), xmm2->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm3_off), xmm3->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm4_off), xmm4->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm5_off), xmm5->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm6_off), xmm6->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm7_off), xmm7->as_VMReg());

  // %%% This is really a waste but we'll keep things as they were for now

  if (true) {

#define NEXTREG(x) (x)->as_VMReg()->next()

    map->set_callee_saved(STACK_OFFSET(st0H_off), NEXTREG(as_FloatRegister(0)));

    map->set_callee_saved(STACK_OFFSET(st1H_off), NEXTREG(as_FloatRegister(1)));

    map->set_callee_saved(STACK_OFFSET(st2H_off), NEXTREG(as_FloatRegister(2)));

    map->set_callee_saved(STACK_OFFSET(st3H_off), NEXTREG(as_FloatRegister(3)));

    map->set_callee_saved(STACK_OFFSET(st4H_off), NEXTREG(as_FloatRegister(4)));

    map->set_callee_saved(STACK_OFFSET(st5H_off), NEXTREG(as_FloatRegister(5)));

    map->set_callee_saved(STACK_OFFSET(st6H_off), NEXTREG(as_FloatRegister(6)));

    map->set_callee_saved(STACK_OFFSET(st7H_off), NEXTREG(as_FloatRegister(7)));

    map->set_callee_saved(STACK_OFFSET(xmm0H_off), NEXTREG(xmm0));

    map->set_callee_saved(STACK_OFFSET(xmm1H_off), NEXTREG(xmm1));

    map->set_callee_saved(STACK_OFFSET(xmm2H_off), NEXTREG(xmm2));

    map->set_callee_saved(STACK_OFFSET(xmm3H_off), NEXTREG(xmm3));

    map->set_callee_saved(STACK_OFFSET(xmm4H_off), NEXTREG(xmm4));

    map->set_callee_saved(STACK_OFFSET(xmm5H_off), NEXTREG(xmm5));

    map->set_callee_saved(STACK_OFFSET(xmm6H_off), NEXTREG(xmm6));

    map->set_callee_saved(STACK_OFFSET(xmm7H_off), NEXTREG(xmm7));

#undef NEXTREG

#undef STACK_OFFSET

  }

  return map;

}

void RegisterSaver::restore_live_registers(MacroAssembler* masm, bool restore_vectors) {

  // Recover XMM & FPU state

  int additional_frame_bytes = 0;

#ifdef COMPILER2

  if (restore_vectors) {

    assert(UseAVX > 0, "256bit vectors are supported only with AVX");

    assert(MaxVectorSize == 32, "only 256bit vectors are supported now");

    additional_frame_bytes = 128;

  }

#else

  assert(!restore_vectors, "vectors are generated only by C2");

#endif

  if (UseSSE == 1) {

    assert(additional_frame_bytes == 0, "");

    __ movflt(xmm0,Address(rsp,xmm0_off*wordSize));

    __ movflt(xmm1,Address(rsp,xmm1_off*wordSize));

    __ movflt(xmm2,Address(rsp,xmm2_off*wordSize));

    __ movflt(xmm3,Address(rsp,xmm3_off*wordSize));

    __ movflt(xmm4,Address(rsp,xmm4_off*wordSize));

    __ movflt(xmm5,Address(rsp,xmm5_off*wordSize));

    __ movflt(xmm6,Address(rsp,xmm6_off*wordSize));

    __ movflt(xmm7,Address(rsp,xmm7_off*wordSize));

  } else if (UseSSE >= 2) {

#define STACK_ADDRESS(x) Address(rsp,(x)*wordSize + additional_frame_bytes)

    __ movdqu(xmm0,STACK_ADDRESS(xmm0_off));

    __ movdqu(xmm1,STACK_ADDRESS(xmm1_off));

    __ movdqu(xmm2,STACK_ADDRESS(xmm2_off));

    __ movdqu(xmm3,STACK_ADDRESS(xmm3_off));

    __ movdqu(xmm4,STACK_ADDRESS(xmm4_off));

    __ movdqu(xmm5,STACK_ADDRESS(xmm5_off));

    __ movdqu(xmm6,STACK_ADDRESS(xmm6_off));

    __ movdqu(xmm7,STACK_ADDRESS(xmm7_off));

#undef STACK_ADDRESS

  }

  if (restore_vectors) {

    // Restore upper half of YMM registes.

    assert(additional_frame_bytes == 128, "");

    __ vinsertf128h(xmm0, Address(rsp,  0));

    __ vinsertf128h(xmm1, Address(rsp, 16));

    __ vinsertf128h(xmm2, Address(rsp, 32));

    __ vinsertf128h(xmm3, Address(rsp, 48));

    __ vinsertf128h(xmm4, Address(rsp, 64));

    __ vinsertf128h(xmm5, Address(rsp, 80));

    __ vinsertf128h(xmm6, Address(rsp, 96));

    __ vinsertf128h(xmm7, Address(rsp,112));

    __ addptr(rsp, additional_frame_bytes);

  }

  __ pop_FPU_state();

  __ addptr(rsp, FPU_regs_live*wordSize); // Pop FPU registers

  __ popf();

  __ popa();

  // Get the rbp, described implicitly by the frame sender code (no oopMap)

  __ pop(rbp);

}

void RegisterSaver::restore_result_registers(MacroAssembler* masm) {

  // Just restore result register. Only used by deoptimization. By

  // now any callee save register that needs to be restore to a c2

  // caller of the deoptee has been extracted into the vframeArray

  // and will be stuffed into the c2i adapter we create for later

  // restoration so only result registers need to be restored here.

  //

  __ frstor(Address(rsp, 0));      // Restore fpu state

  // Recover XMM & FPU state

  if( UseSSE == 1 ) {

    __ movflt(xmm0, Address(rsp, xmm0_off*wordSize));

  } else if( UseSSE >= 2 ) {

    __ movdbl(xmm0, Address(rsp, xmm0_off*wordSize));

  }

  __ movptr(rax, Address(rsp, rax_off*wordSize));

  __ movptr(rdx, Address(rsp, rdx_off*wordSize));

  // Pop all of the register save are off the stack except the return address

  __ addptr(rsp, return_off * wordSize);

}

// Is vector's size (in bytes) bigger than a size saved by default?

// 16 bytes XMM registers are saved by default using SSE2 movdqu instructions.

// Note, MaxVectorSize == 0 with UseSSE < 2 and vectors are not generated.

bool SharedRuntime::is_wide_vector(int size) {

  return size > 16;

}

// The java_calling_convention describes stack locations as ideal slots on

// a frame with no abi restrictions. Since we must observe abi restrictions

// (like the placement of the register window) the slots must be biased by

// the following value.

static int reg2offset_in(VMReg r) {

  // Account for saved rbp, and return address

  // This should really be in_preserve_stack_slots

  return (r->reg2stack() + 2) * VMRegImpl::stack_slot_size;

}

static int reg2offset_out(VMReg r) {

  return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;

}

// ---------------------------------------------------------------------------

// Read the array of BasicTypes from a signature, and compute where the

// arguments should go.  Values in the VMRegPair regs array refer to 4-byte

// quantities.  Values less than SharedInfo::stack0 are registers, those above

// refer to 4-byte stack slots.  All stack slots are based off of the stack pointer

// as framesizes are fixed.

// VMRegImpl::stack0 refers to the first slot 0(sp).

// and VMRegImpl::stack0+1 refers to the memory word 4-byes higher.  Register

// up to RegisterImpl::number_of_registers) are the 32-bit

// integer registers.

// Pass first two oop/int args in registers ECX and EDX.

// Pass first two float/double args in registers XMM0 and XMM1.

// Doubles have precedence, so if you pass a mix of floats and doubles

// the doubles will grab the registers before the floats will.

// Note: the INPUTS in sig_bt are in units of Java argument words, which are

// either 32-bit or 64-bit depending on the build.  The OUTPUTS are in 32-bit

// units regardless of build. Of course for i486 there is no 64 bit build

// ---------------------------------------------------------------------------

// The compiled Java calling convention.

// Pass first two oop/int args in registers ECX and EDX.

// Pass first two float/double args in registers XMM0 and XMM1.

// Doubles have precedence, so if you pass a mix of floats and doubles

// the doubles will grab the registers before the floats will.

int SharedRuntime::java_calling_convention(const BasicType *sig_bt,

                                           VMRegPair *regs,

                                           int total_args_passed,

                                           int is_outgoing) {

  uint    stack = 0;          // Starting stack position for args on stack

  // Pass first two oop/int args in registers ECX and EDX.

  uint reg_arg0 = 9999;

  uint reg_arg1 = 9999;

  // Pass first two float/double args in registers XMM0 and XMM1.

  // Doubles have precedence, so if you pass a mix of floats and doubles

  // the doubles will grab the registers before the floats will.

  // CNC - TURNED OFF FOR non-SSE.

  //       On Intel we have to round all doubles (and most floats) at

  //       call sites by storing to the stack in any case.

  // UseSSE=0 ==> Don't Use ==> 9999+0

  // UseSSE=1 ==> Floats only ==> 9999+1

  // UseSSE>=2 ==> Floats or doubles ==> 9999+2

  enum { fltarg_dontuse = 9999+0, fltarg_float_only = 9999+1, fltarg_flt_dbl = 9999+2 };

  uint fargs = (UseSSE>=2) ? 2 : UseSSE;

  uint freg_arg0 = 9999+fargs;

  uint freg_arg1 = 9999+fargs;

  // Pass doubles & longs aligned on the stack.  First count stack slots for doubles

  int i;

  for( i = 0; i < total_args_passed; i++) {

    if( sig_bt[i] == T_DOUBLE ) {

      // first 2 doubles go in registers

      if( freg_arg0 == fltarg_flt_dbl ) freg_arg0 = i;

      else if( freg_arg1 == fltarg_flt_dbl ) freg_arg1 = i;

      else // Else double is passed low on the stack to be aligned.

        stack += 2;

    } else if( sig_bt[i] == T_LONG ) {

      stack += 2;

    }

  }

  int dstack = 0;             // Separate counter for placing doubles

  // Now pick where all else goes.

  for( i = 0; i < total_args_passed; i++) {

    // From the type and the argument number (count) compute the location

    switch( sig_bt[i] ) {

    case T_SHORT:

    case T_CHAR:

    case T_BYTE:

    case T_BOOLEAN:

    case T_INT:

    case T_ARRAY:

    case T_OBJECT:

    case T_ADDRESS:

      if( reg_arg0 == 9999 )  {

        reg_arg0 = i;

        regs[i].set1(rcx->as_VMReg());

      } else if( reg_arg1 == 9999 )  {

        reg_arg1 = i;

        regs[i].set1(rdx->as_VMReg());

      } else {

        regs[i].set1(VMRegImpl::stack2reg(stack++));

      }

      break;

    case T_FLOAT:

      if( freg_arg0 == fltarg_flt_dbl || freg_arg0 == fltarg_float_only ) {

        freg_arg0 = i;

        regs[i].set1(xmm0->as_VMReg());

      } else if( freg_arg1 == fltarg_flt_dbl || freg_arg1 == fltarg_float_only ) {

        freg_arg1 = i;

        regs[i].set1(xmm1->as_VMReg());

      } else {

        regs[i].set1(VMRegImpl::stack2reg(stack++));

      }

      break;

    case T_LONG:

      assert(sig_bt[i+1] == T_VOID, "missing Half" );

      regs[i].set2(VMRegImpl::stack2reg(dstack));

      dstack += 2;

      break;

    case T_DOUBLE:

      assert(sig_bt[i+1] == T_VOID, "missing Half" );

      if( freg_arg0 == (uint)i ) {

        regs[i].set2(xmm0->as_VMReg());

      } else if( freg_arg1 == (uint)i ) {

        regs[i].set2(xmm1->as_VMReg());

      } else {

        regs[i].set2(VMRegImpl::stack2reg(dstack));

        dstack += 2;

      }

      break;

    case T_VOID: regs[i].set_bad(); break;

      break;

    default:

      ShouldNotReachHere();

      break;

    }

  }

  // return value can be odd number of VMRegImpl stack slots make multiple of 2

  return round_to(stack, 2);

}

// Patch the callers callsite with entry to compiled code if it exists.

static void patch_callers_callsite(MacroAssembler *masm) {

  Label L;

  __ cmpptr(Address(rbx, in_bytes(Method::code_offset())), (int32_t)NULL_WORD);

  __ jcc(Assembler::equal, L);

  // Schedule the branch target address early.

  // Call into the VM to patch the caller, then jump to compiled callee