/*

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,

 * CA 95054 USA or visit www.sun.com if you need additional information or

 * have any questions.

 *

 */

#include "incls/_precompiled.incl"

#include "incls/_sharedRuntime_x86_32.cpp.incl"

#define __ masm->

#ifdef COMPILER2

UncommonTrapBlob   *SharedRuntime::_uncommon_trap_blob;

#endif // COMPILER2

DeoptimizationBlob *SharedRuntime::_deopt_blob;

SafepointBlob      *SharedRuntime::_polling_page_safepoint_handler_blob;

SafepointBlob      *SharedRuntime::_polling_page_return_handler_blob;

RuntimeStub*       SharedRuntime::_wrong_method_blob;

RuntimeStub*       SharedRuntime::_ic_miss_blob;

RuntimeStub*       SharedRuntime::_resolve_opt_virtual_call_blob;

RuntimeStub*       SharedRuntime::_resolve_virtual_call_blob;

RuntimeStub*       SharedRuntime::_resolve_static_call_blob;

const int StackAlignmentInSlots = StackAlignmentInBytes / VMRegImpl::stack_slot_size;

class RegisterSaver {

  enum { FPU_regs_live = 8 /*for the FPU stack*/+8/*eight more for XMM registers*/ };

  // Capture info about frame layout

  enum layout {

                fpu_state_off = 0,

                fpu_state_end = fpu_state_off+FPUStateSizeInWords-1,

                st0_off, st0H_off,

                st1_off, st1H_off,

                st2_off, st2H_off,

                st3_off, st3H_off,

                st4_off, st4H_off,

                st5_off, st5H_off,

                st6_off, st6H_off,

                st7_off, st7H_off,

                xmm0_off, xmm0H_off,

                xmm1_off, xmm1H_off,

                xmm2_off, xmm2H_off,

                xmm3_off, xmm3H_off,

                xmm4_off, xmm4H_off,

                xmm5_off, xmm5H_off,

                xmm6_off, xmm6H_off,

                xmm7_off, xmm7H_off,

                flags_off,

                rdi_off,

                rsi_off,

                ignore_off,  // extra copy of rbp,

                rsp_off,

                rbx_off,

                rdx_off,

                rcx_off,

                rax_off,

                // The frame sender code expects that rbp will be in the "natural" place and

                // will override any oopMap setting for it. We must therefore force the layout

                // so that it agrees with the frame sender code.

                rbp_off,

                return_off,      // slot for return address

                reg_save_size };

  public:

  static OopMap* save_live_registers(MacroAssembler* masm, int additional_frame_words,

                                     int* total_frame_words, bool verify_fpu = true);

  static void restore_live_registers(MacroAssembler* masm);

  static int rax_offset() { return rax_off; }

  static int rbx_offset() { return rbx_off; }

  // Offsets into the register save area

  // Used by deoptimization when it is managing result register

  // values on its own

  static int raxOffset(void) { return rax_off; }

  static int rdxOffset(void) { return rdx_off; }

  static int rbxOffset(void) { return rbx_off; }

  static int xmm0Offset(void) { return xmm0_off; }

  // This really returns a slot in the fp save area, which one is not important

  static int fpResultOffset(void) { return st0_off; }

  // During deoptimization only the result register need to be restored

  // all the other values have already been extracted.

  static void restore_result_registers(MacroAssembler* masm);

};

OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, int additional_frame_words,

                                           int* total_frame_words, bool verify_fpu) {

  int frame_size_in_bytes =  (reg_save_size + additional_frame_words) * wordSize;

  int frame_words = frame_size_in_bytes / wordSize;

  *total_frame_words = frame_words;

  assert(FPUStateSizeInWords == 27, "update stack layout");

  // save registers, fpu state, and flags

  // We assume caller has already has return address slot on the stack

  // We push epb twice in this sequence because we want the real rbp,

  // to be under the return like a normal enter and we want to use pusha

  // We push by hand instead of pusing push

  __ enter();

  __ pusha();

  __ pushf();

  __ subptr(rsp,FPU_regs_live*sizeof(jdouble)); // Push FPU registers space

  __ push_FPU_state();          // Save FPU state & init

  if (verify_fpu) {

    // Some stubs may have non standard FPU control word settings so

    // only check and reset the value when it required to be the

    // standard value.  The safepoint blob in particular can be used

    // in methods which are using the 24 bit control word for

    // optimized float math.

#ifdef ASSERT

    // Make sure the control word has the expected value

    Label ok;

    __ cmpw(Address(rsp, 0), StubRoutines::fpu_cntrl_wrd_std());

    __ jccb(Assembler::equal, ok);

    __ stop("corrupted control word detected");

    __ bind(ok);

#endif

    // Reset the control word to guard against exceptions being unmasked

    // since fstp_d can cause FPU stack underflow exceptions.  Write it

    // into the on stack copy and then reload that to make sure that the

    // current and future values are correct.

    __ movw(Address(rsp, 0), StubRoutines::fpu_cntrl_wrd_std());

  }

  __ frstor(Address(rsp, 0));

  if (!verify_fpu) {

    // Set the control word so that exceptions are masked for the

    // following code.

    __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));

  }

  // Save the FPU registers in de-opt-able form

  __ fstp_d(Address(rsp, st0_off*wordSize)); // st(0)

  __ fstp_d(Address(rsp, st1_off*wordSize)); // st(1)

  __ fstp_d(Address(rsp, st2_off*wordSize)); // st(2)

  __ fstp_d(Address(rsp, st3_off*wordSize)); // st(3)

  __ fstp_d(Address(rsp, st4_off*wordSize)); // st(4)

  __ fstp_d(Address(rsp, st5_off*wordSize)); // st(5)

  __ fstp_d(Address(rsp, st6_off*wordSize)); // st(6)

  __ fstp_d(Address(rsp, st7_off*wordSize)); // st(7)

  if( UseSSE == 1 ) {           // Save the XMM state

    __ movflt(Address(rsp,xmm0_off*wordSize),xmm0);

    __ movflt(Address(rsp,xmm1_off*wordSize),xmm1);

    __ movflt(Address(rsp,xmm2_off*wordSize),xmm2);

    __ movflt(Address(rsp,xmm3_off*wordSize),xmm3);

    __ movflt(Address(rsp,xmm4_off*wordSize),xmm4);

    __ movflt(Address(rsp,xmm5_off*wordSize),xmm5);

    __ movflt(Address(rsp,xmm6_off*wordSize),xmm6);

    __ movflt(Address(rsp,xmm7_off*wordSize),xmm7);

  } else if( UseSSE >= 2 ) {

    __ movdbl(Address(rsp,xmm0_off*wordSize),xmm0);

    __ movdbl(Address(rsp,xmm1_off*wordSize),xmm1);

    __ movdbl(Address(rsp,xmm2_off*wordSize),xmm2);

    __ movdbl(Address(rsp,xmm3_off*wordSize),xmm3);

    __ movdbl(Address(rsp,xmm4_off*wordSize),xmm4);

    __ movdbl(Address(rsp,xmm5_off*wordSize),xmm5);

    __ movdbl(Address(rsp,xmm6_off*wordSize),xmm6);

    __ movdbl(Address(rsp,xmm7_off*wordSize),xmm7);

  }

  // Set an oopmap for the call site.  This oopmap will map all

  // oop-registers and debug-info registers as callee-saved.  This

  // will allow deoptimization at this safepoint to find all possible

  // debug-info recordings, as well as let GC find all oops.

  OopMapSet *oop_maps = new OopMapSet();

  OopMap* map =  new OopMap( frame_words, 0 );

#define STACK_OFFSET(x) VMRegImpl::stack2reg((x) + additional_frame_words)

  map->set_callee_saved(STACK_OFFSET( rax_off), rax->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( rcx_off), rcx->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( rdx_off), rdx->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( rbx_off), rbx->as_VMReg());

  // rbp, location is known implicitly, no oopMap

  map->set_callee_saved(STACK_OFFSET( rsi_off), rsi->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( rdi_off), rdi->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(st0_off), as_FloatRegister(0)->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(st1_off), as_FloatRegister(1)->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(st2_off), as_FloatRegister(2)->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(st3_off), as_FloatRegister(3)->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(st4_off), as_FloatRegister(4)->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(st5_off), as_FloatRegister(5)->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(st6_off), as_FloatRegister(6)->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(st7_off), as_FloatRegister(7)->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm0_off), xmm0->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm1_off), xmm1->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm2_off), xmm2->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm3_off), xmm3->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm4_off), xmm4->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm5_off), xmm5->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm6_off), xmm6->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(xmm7_off), xmm7->as_VMReg());

  // %%% This is really a waste but we'll keep things as they were for now

  if (true) {

#define NEXTREG(x) (x)->as_VMReg()->next()

    map->set_callee_saved(STACK_OFFSET(st0H_off), NEXTREG(as_FloatRegister(0)));

    map->set_callee_saved(STACK_OFFSET(st1H_off), NEXTREG(as_FloatRegister(1)));

    map->set_callee_saved(STACK_OFFSET(st2H_off), NEXTREG(as_FloatRegister(2)));

    map->set_callee_saved(STACK_OFFSET(st3H_off), NEXTREG(as_FloatRegister(3)));

    map->set_callee_saved(STACK_OFFSET(st4H_off), NEXTREG(as_FloatRegister(4)));

    map->set_callee_saved(STACK_OFFSET(st5H_off), NEXTREG(as_FloatRegister(5)));

    map->set_callee_saved(STACK_OFFSET(st6H_off), NEXTREG(as_FloatRegister(6)));

    map->set_callee_saved(STACK_OFFSET(st7H_off), NEXTREG(as_FloatRegister(7)));

    map->set_callee_saved(STACK_OFFSET(xmm0H_off), NEXTREG(xmm0));

    map->set_callee_saved(STACK_OFFSET(xmm1H_off), NEXTREG(xmm1));

    map->set_callee_saved(STACK_OFFSET(xmm2H_off), NEXTREG(xmm2));

    map->set_callee_saved(STACK_OFFSET(xmm3H_off), NEXTREG(xmm3));

    map->set_callee_saved(STACK_OFFSET(xmm4H_off), NEXTREG(xmm4));

    map->set_callee_saved(STACK_OFFSET(xmm5H_off), NEXTREG(xmm5));

    map->set_callee_saved(STACK_OFFSET(xmm6H_off), NEXTREG(xmm6));

    map->set_callee_saved(STACK_OFFSET(xmm7H_off), NEXTREG(xmm7));

#undef NEXTREG

#undef STACK_OFFSET

  }

  return map;

}

void RegisterSaver::restore_live_registers(MacroAssembler* masm) {

  // Recover XMM & FPU state

  if( UseSSE == 1 ) {

    __ movflt(xmm0,Address(rsp,xmm0_off*wordSize));

    __ movflt(xmm1,Address(rsp,xmm1_off*wordSize));

    __ movflt(xmm2,Address(rsp,xmm2_off*wordSize));

    __ movflt(xmm3,Address(rsp,xmm3_off*wordSize));

    __ movflt(xmm4,Address(rsp,xmm4_off*wordSize));

    __ movflt(xmm5,Address(rsp,xmm5_off*wordSize));

    __ movflt(xmm6,Address(rsp,xmm6_off*wordSize));

    __ movflt(xmm7,Address(rsp,xmm7_off*wordSize));

  } else if( UseSSE >= 2 ) {

    __ movdbl(xmm0,Address(rsp,xmm0_off*wordSize));

    __ movdbl(xmm1,Address(rsp,xmm1_off*wordSize));

    __ movdbl(xmm2,Address(rsp,xmm2_off*wordSize));

    __ movdbl(xmm3,Address(rsp,xmm3_off*wordSize));

    __ movdbl(xmm4,Address(rsp,xmm4_off*wordSize));

    __ movdbl(xmm5,Address(rsp,xmm5_off*wordSize));

    __ movdbl(xmm6,Address(rsp,xmm6_off*wordSize));

    __ movdbl(xmm7,Address(rsp,xmm7_off*wordSize));

  }

  __ pop_FPU_state();

  __ addptr(rsp, FPU_regs_live*sizeof(jdouble)); // Pop FPU registers

  __ popf();

  __ popa();

  // Get the rbp, described implicitly by the frame sender code (no oopMap)

  __ pop(rbp);

}

void RegisterSaver::restore_result_registers(MacroAssembler* masm) {

  // Just restore result register. Only used by deoptimization. By

  // now any callee save register that needs to be restore to a c2

  // caller of the deoptee has been extracted into the vframeArray

  // and will be stuffed into the c2i adapter we create for later

  // restoration so only result registers need to be restored here.

  //

  __ frstor(Address(rsp, 0));      // Restore fpu state

  // Recover XMM & FPU state

  if( UseSSE == 1 ) {

    __ movflt(xmm0, Address(rsp, xmm0_off*wordSize));

  } else if( UseSSE >= 2 ) {

    __ movdbl(xmm0, Address(rsp, xmm0_off*wordSize));

  }

  __ movptr(rax, Address(rsp, rax_off*wordSize));

  __ movptr(rdx, Address(rsp, rdx_off*wordSize));

  // Pop all of the register save are off the stack except the return address

  __ addptr(rsp, return_off * wordSize);

}

// The java_calling_convention describes stack locations as ideal slots on

// a frame with no abi restrictions. Since we must observe abi restrictions

// (like the placement of the register window) the slots must be biased by

// the following value.

static int reg2offset_in(VMReg r) {

  // Account for saved rbp, and return address

  // This should really be in_preserve_stack_slots

  return (r->reg2stack() + 2) * VMRegImpl::stack_slot_size;

}

static int reg2offset_out(VMReg r) {

  return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;

}

// ---------------------------------------------------------------------------

// Read the array of BasicTypes from a signature, and compute where the

// arguments should go.  Values in the VMRegPair regs array refer to 4-byte

// quantities.  Values less than SharedInfo::stack0 are registers, those above

// refer to 4-byte stack slots.  All stack slots are based off of the stack pointer

// as framesizes are fixed.

// VMRegImpl::stack0 refers to the first slot 0(sp).

// and VMRegImpl::stack0+1 refers to the memory word 4-byes higher.  Register

// up to RegisterImpl::number_of_registers) are the 32-bit

// integer registers.

// Pass first two oop/int args in registers ECX and EDX.

// Pass first two float/double args in registers XMM0 and XMM1.

// Doubles have precedence, so if you pass a mix of floats and doubles

// the doubles will grab the registers before the floats will.

// Note: the INPUTS in sig_bt are in units of Java argument words, which are

// either 32-bit or 64-bit depending on the build.  The OUTPUTS are in 32-bit

// units regardless of build. Of course for i486 there is no 64 bit build

// ---------------------------------------------------------------------------

// The compiled Java calling convention.

// Pass first two oop/int args in registers ECX and EDX.

// Pass first two float/double args in registers XMM0 and XMM1.

// Doubles have precedence, so if you pass a mix of floats and doubles

// the doubles will grab the registers before the floats will.

int SharedRuntime::java_calling_convention(const BasicType *sig_bt,

                                           VMRegPair *regs,

                                           int total_args_passed,

                                           int is_outgoing) {

  uint    stack = 0;          // Starting stack position for args on stack

  // Pass first two oop/int args in registers ECX and EDX.

  uint reg_arg0 = 9999;

  uint reg_arg1 = 9999;

  // Pass first two float/double args in registers XMM0 and XMM1.

  // Doubles have precedence, so if you pass a mix of floats and doubles

  // the doubles will grab the registers before the floats will.

  // CNC - TURNED OFF FOR non-SSE.

  //       On Intel we have to round all doubles (and most floats) at

  //       call sites by storing to the stack in any case.

  // UseSSE=0 ==> Don't Use ==> 9999+0

  // UseSSE=1 ==> Floats only ==> 9999+1

  // UseSSE>=2 ==> Floats or doubles ==> 9999+2

  enum { fltarg_dontuse = 9999+0, fltarg_float_only = 9999+1, fltarg_flt_dbl = 9999+2 };

  uint fargs = (UseSSE>=2) ? 2 : UseSSE;

  uint freg_arg0 = 9999+fargs;

  uint freg_arg1 = 9999+fargs;

  // Pass doubles & longs aligned on the stack.  First count stack slots for doubles

  int i;

  for( i = 0; i < total_args_passed; i++) {

    if( sig_bt[i] == T_DOUBLE ) {

      // first 2 doubles go in registers

      if( freg_arg0 == fltarg_flt_dbl ) freg_arg0 = i;

      else if( freg_arg1 == fltarg_flt_dbl ) freg_arg1 = i;

      else // Else double is passed low on the stack to be aligned.

        stack += 2;

    } else if( sig_bt[i] == T_LONG ) {

      stack += 2;

    }

  }

  int dstack = 0;             // Separate counter for placing doubles

  // Now pick where all else goes.

  for( i = 0; i < total_args_passed; i++) {

    // From the type and the argument number (count) compute the location

    switch( sig_bt[i] ) {

    case T_SHORT:

    case T_CHAR:

    case T_BYTE:

    case T_BOOLEAN:

    case T_INT:

    case T_ARRAY:

    case T_OBJECT:

    case T_ADDRESS:

      if( reg_arg0 == 9999 )  {

        reg_arg0 = i;

        regs[i].set1(rcx->as_VMReg());

      } else if( reg_arg1 == 9999 )  {

        reg_arg1 = i;

        regs[i].set1(rdx->as_VMReg());

      } else {

        regs[i].set1(VMRegImpl::stack2reg(stack++));

      }

      break;

    case T_FLOAT:

      if( freg_arg0 == fltarg_flt_dbl || freg_arg0 == fltarg_float_only ) {

        freg_arg0 = i;

        regs[i].set1(xmm0->as_VMReg());

      } else if( freg_arg1 == fltarg_flt_dbl || freg_arg1 == fltarg_float_only ) {

        freg_arg1 = i;

        regs[i].set1(xmm1->as_VMReg());

      } else {

        regs[i].set1(VMRegImpl::stack2reg(stack++));

      }

      break;

    case T_LONG:

      assert(sig_bt[i+1] == T_VOID, "missing Half" );

      regs[i].set2(VMRegImpl::stack2reg(dstack));

      dstack += 2;

      break;

    case T_DOUBLE:

      assert(sig_bt[i+1] == T_VOID, "missing Half" );

      if( freg_arg0 == (uint)i ) {

        regs[i].set2(xmm0->as_VMReg());

      } else if( freg_arg1 == (uint)i ) {

        regs[i].set2(xmm1->as_VMReg());

      } else {

        regs[i].set2(VMRegImpl::stack2reg(dstack));

        dstack += 2;

      }

      break;

    case T_VOID: regs[i].set_bad(); break;

      break;

    default:

      ShouldNotReachHere();

      break;

    }

  }

  // return value can be odd number of VMRegImpl stack slots make multiple of 2

  return round_to(stack, 2);

}

// Patch the callers callsite with entry to compiled code if it exists.

static void patch_callers_callsite(MacroAssembler *masm) {

  Label L;

  __ verify_oop(rbx);

  __ cmpptr(Address(rbx, in_bytes(methodOopDesc::code_offset())), (int32_t)NULL_WORD);

  __ jcc(Assembler::equal, L);

  // Schedule the branch target address early.

  // Call into the VM to patch the caller, then jump to compiled callee

  // rax, isn't live so capture return address while we easily can

  __ movptr(rax, Address(rsp, 0));

  __ pusha();

  __ pushf();

  if (UseSSE == 1) {

    __ subptr(rsp, 2*wordSize);

    __ movflt(Address(rsp, 0), xmm0);

    __ movflt(Address(rsp, wordSize), xmm1);

  }

  if (UseSSE >= 2) {

    __ subptr(rsp, 4*wordSize);

    __ movdbl(Address(rsp, 0), xmm0);

    __ movdbl(Address(rsp, 2*wordSize), xmm1);

  }

#ifdef COMPILER2

  // C2 may leave the stack dirty if not in SSE2+ mode

  if (UseSSE >= 2) {

    __ verify_FPU(0, "c2i transition should have clean FPU stack");

  } else {

    __ empty_FPU_stack();

  }

#endif /* COMPILER2 */

  // VM needs caller's callsite

  __ push(rax);

  // VM needs target method

  __ push(rbx);

  __ verify_oop(rbx);

  __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite)));

  __ addptr(rsp, 2*wordSize);

  if (UseSSE == 1) {

    __ movflt(xmm0, Address(rsp, 0));

    __ movflt(xmm1, Address(rsp, wordSize));

    __ addptr(rsp, 2*wordSize);

  }

  if (UseSSE >= 2) {

    __ movdbl(xmm0, Address(rsp, 0));

    __ movdbl(xmm1, Address(rsp, 2*wordSize));

    __ addptr(rsp, 4*wordSize);

  }

  __ popf();

  __ popa();

  __ bind(L);

}

// Helper function to put tags in interpreter stack.

static void  tag_stack(MacroAssembler *masm, const BasicType sig, int st_off) {

  if (TaggedStackInterpreter) {

    int tag_offset = st_off + Interpreter::expr_tag_offset_in_bytes(0);

    if (sig == T_OBJECT || sig == T_ARRAY) {

      __ movptr(Address(rsp, tag_offset), frame::TagReference);

    } else if (sig == T_LONG || sig == T_DOUBLE) {

      int next_tag_offset = st_off + Interpreter::expr_tag_offset_in_bytes(1);

      __ movptr(Address(rsp, next_tag_offset), frame::TagValue);

      __ movptr(Address(rsp, tag_offset), frame::TagValue);

    } else {

      __ movptr(Address(rsp, tag_offset), frame::TagValue);

    }

  }

}

// Double and long values with Tagged stacks are not contiguous.

static void move_c2i_double(MacroAssembler *masm, XMMRegister r, int st_off) {

  int next_off = st_off - Interpreter::stackElementSize();