/*

 * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "asm/macroAssembler.hpp"

#include "asm/macroAssembler.inline.hpp"

#include "code/debugInfoRec.hpp"

#include "code/icBuffer.hpp"

#include "code/nativeInst.hpp"

#include "code/vtableStubs.hpp"

#include "gc/shared/gcLocker.hpp"

#include "interpreter/interpreter.hpp"

#include "logging/log.hpp"

#include "memory/resourceArea.hpp"

#include "oops/compiledICHolder.hpp"

#include "oops/klass.inline.hpp"

#include "runtime/safepointMechanism.hpp"

#include "runtime/sharedRuntime.hpp"

#include "runtime/vframeArray.hpp"

#include "utilities/align.hpp"

#include "vmreg_x86.inline.hpp"

#ifdef COMPILER1

#include "c1/c1_Runtime1.hpp"

#endif

#ifdef COMPILER2

#include "opto/runtime.hpp"

#endif

#define __ masm->

const int StackAlignmentInSlots = StackAlignmentInBytes / VMRegImpl::stack_slot_size;

class RegisterSaver {

  // Capture info about frame layout

#define DEF_XMM_OFFS(regnum) xmm ## regnum ## _off = xmm_off + (regnum)*16/BytesPerInt, xmm ## regnum ## H_off

  enum layout {

                fpu_state_off = 0,

                fpu_state_end = fpu_state_off+FPUStateSizeInWords,

                st0_off, st0H_off,

                st1_off, st1H_off,

                st2_off, st2H_off,

                st3_off, st3H_off,

                st4_off, st4H_off,

                st5_off, st5H_off,

                st6_off, st6H_off,

                st7_off, st7H_off,

                xmm_off,

                DEF_XMM_OFFS(0),

                DEF_XMM_OFFS(1),

                DEF_XMM_OFFS(2),

                DEF_XMM_OFFS(3),

                DEF_XMM_OFFS(4),

                DEF_XMM_OFFS(5),

                DEF_XMM_OFFS(6),

                DEF_XMM_OFFS(7),

                flags_off = xmm7_off + 16/BytesPerInt + 1, // 16-byte stack alignment fill word

                rdi_off,

                rsi_off,

                ignore_off,  // extra copy of rbp,

                rsp_off,

                rbx_off,

                rdx_off,

                rcx_off,

                rax_off,

                // The frame sender code expects that rbp will be in the "natural" place and

                // will override any oopMap setting for it. We must therefore force the layout

                // so that it agrees with the frame sender code.

                rbp_off,

                return_off,      // slot for return address

                reg_save_size };

  enum { FPU_regs_live = flags_off - fpu_state_end };

  public:

  static OopMap* save_live_registers(MacroAssembler* masm, int additional_frame_words,

                                     int* total_frame_words, bool verify_fpu = true, bool save_vectors = false);

  static void restore_live_registers(MacroAssembler* masm, bool restore_vectors = false);

  static int rax_offset() { return rax_off; }

  static int rbx_offset() { return rbx_off; }

  // Offsets into the register save area

  // Used by deoptimization when it is managing result register

  // values on its own

  static int raxOffset(void) { return rax_off; }

  static int rdxOffset(void) { return rdx_off; }

  static int rbxOffset(void) { return rbx_off; }

  static int xmm0Offset(void) { return xmm0_off; }

  // This really returns a slot in the fp save area, which one is not important

  static int fpResultOffset(void) { return st0_off; }

  // During deoptimization only the result register need to be restored

  // all the other values have already been extracted.

  static void restore_result_registers(MacroAssembler* masm);

};

OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, int additional_frame_words,

                                           int* total_frame_words, bool verify_fpu, bool save_vectors) {

  int num_xmm_regs = XMMRegisterImpl::number_of_registers;

  int ymm_bytes = num_xmm_regs * 16;

  int zmm_bytes = num_xmm_regs * 32;

#ifdef COMPILER2

  if (save_vectors) {

    assert(UseAVX > 0, "Vectors larger than 16 byte long are supported only with AVX");

    assert(MaxVectorSize <= 64, "Only up to 64 byte long vectors are supported");

    // Save upper half of YMM registers

    int vect_bytes = ymm_bytes;

    if (UseAVX > 2) {

      // Save upper half of ZMM registers as well

      vect_bytes += zmm_bytes;

    }

    additional_frame_words += vect_bytes / wordSize;

  }

#else

  assert(!save_vectors, "vectors are generated only by C2");

#endif

  int frame_size_in_bytes = (reg_save_size + additional_frame_words) * wordSize;

  int frame_words = frame_size_in_bytes / wordSize;

  *total_frame_words = frame_words;

  assert(FPUStateSizeInWords == 27, "update stack layout");

  // save registers, fpu state, and flags

  // We assume caller has already has return address slot on the stack

  // We push epb twice in this sequence because we want the real rbp,

  // to be under the return like a normal enter and we want to use pusha

  // We push by hand instead of using push.

  __ enter();

  __ pusha();

  __ pushf();

  __ subptr(rsp,FPU_regs_live*wordSize); // Push FPU registers space

  __ push_FPU_state();          // Save FPU state & init

  if (verify_fpu) {

    // Some stubs may have non standard FPU control word settings so

    // only check and reset the value when it required to be the

    // standard value.  The safepoint blob in particular can be used

    // in methods which are using the 24 bit control word for

    // optimized float math.

#ifdef ASSERT

    // Make sure the control word has the expected value

    Label ok;

    __ cmpw(Address(rsp, 0), StubRoutines::fpu_cntrl_wrd_std());

    __ jccb(Assembler::equal, ok);

    __ stop("corrupted control word detected");

    __ bind(ok);

#endif

    // Reset the control word to guard against exceptions being unmasked

    // since fstp_d can cause FPU stack underflow exceptions.  Write it

    // into the on stack copy and then reload that to make sure that the

    // current and future values are correct.

    __ movw(Address(rsp, 0), StubRoutines::fpu_cntrl_wrd_std());

  }

  __ frstor(Address(rsp, 0));

  if (!verify_fpu) {

    // Set the control word so that exceptions are masked for the

    // following code.

    __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));

  }

  int off = st0_off;

  int delta = st1_off - off;

  // Save the FPU registers in de-opt-able form

  for (int n = 0; n < FloatRegisterImpl::number_of_registers; n++) {

    __ fstp_d(Address(rsp, off*wordSize));

    off += delta;

  }

  off = xmm0_off;

  delta = xmm1_off - off;

  if(UseSSE == 1) {

    // Save the XMM state

    for (int n = 0; n < num_xmm_regs; n++) {

      __ movflt(Address(rsp, off*wordSize), as_XMMRegister(n));

      off += delta;

    }

  } else if(UseSSE >= 2) {

    // Save whole 128bit (16 bytes) XMM registers

    for (int n = 0; n < num_xmm_regs; n++) {

      __ movdqu(Address(rsp, off*wordSize), as_XMMRegister(n));

      off += delta;

    }

  }

  if (save_vectors) {

    __ subptr(rsp, ymm_bytes);

    // Save upper half of YMM registers

    for (int n = 0; n < num_xmm_regs; n++) {

      __ vextractf128_high(Address(rsp, n*16), as_XMMRegister(n));

    }

    if (UseAVX > 2) {

      __ subptr(rsp, zmm_bytes);

      // Save upper half of ZMM registers

      for (int n = 0; n < num_xmm_regs; n++) {

        __ vextractf64x4_high(Address(rsp, n*32), as_XMMRegister(n));

      }

    }

  }

  __ vzeroupper();

  // Set an oopmap for the call site.  This oopmap will map all

  // oop-registers and debug-info registers as callee-saved.  This

  // will allow deoptimization at this safepoint to find all possible

  // debug-info recordings, as well as let GC find all oops.

  OopMapSet *oop_maps = new OopMapSet();

  OopMap* map =  new OopMap( frame_words, 0 );

#define STACK_OFFSET(x) VMRegImpl::stack2reg((x) + additional_frame_words)

#define NEXTREG(x) (x)->as_VMReg()->next()

  map->set_callee_saved(STACK_OFFSET(rax_off), rax->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(rcx_off), rcx->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(rdx_off), rdx->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(rbx_off), rbx->as_VMReg());

  // rbp, location is known implicitly, no oopMap

  map->set_callee_saved(STACK_OFFSET(rsi_off), rsi->as_VMReg());

  map->set_callee_saved(STACK_OFFSET(rdi_off), rdi->as_VMReg());

  // %%% This is really a waste but we'll keep things as they were for now for the upper component

  off = st0_off;

  delta = st1_off - off;

  for (int n = 0; n < FloatRegisterImpl::number_of_registers; n++) {

    FloatRegister freg_name = as_FloatRegister(n);

    map->set_callee_saved(STACK_OFFSET(off), freg_name->as_VMReg());

    map->set_callee_saved(STACK_OFFSET(off+1), NEXTREG(freg_name));

    off += delta;

  }

  off = xmm0_off;

  delta = xmm1_off - off;

  for (int n = 0; n < num_xmm_regs; n++) {

    XMMRegister xmm_name = as_XMMRegister(n);

    map->set_callee_saved(STACK_OFFSET(off), xmm_name->as_VMReg());

    map->set_callee_saved(STACK_OFFSET(off+1), NEXTREG(xmm_name));

    off += delta;

  }

#undef NEXTREG

#undef STACK_OFFSET

  return map;

}

void RegisterSaver::restore_live_registers(MacroAssembler* masm, bool restore_vectors) {

  int num_xmm_regs = XMMRegisterImpl::number_of_registers;

  int ymm_bytes = num_xmm_regs * 16;

  int zmm_bytes = num_xmm_regs * 32;

  // Recover XMM & FPU state

  int additional_frame_bytes = 0;

#ifdef COMPILER2

  if (restore_vectors) {

    assert(UseAVX > 0, "Vectors larger than 16 byte long are supported only with AVX");

    assert(MaxVectorSize <= 64, "Only up to 64 byte long vectors are supported");

    // Save upper half of YMM registers

    additional_frame_bytes = ymm_bytes;

    if (UseAVX > 2) {

      // Save upper half of ZMM registers as well

      additional_frame_bytes += zmm_bytes;

    }

  }

#else

  assert(!restore_vectors, "vectors are generated only by C2");

#endif

  int off = xmm0_off;

  int delta = xmm1_off - off;

  __ vzeroupper();

  if (UseSSE == 1) {

    // Restore XMM registers

    assert(additional_frame_bytes == 0, "");

    for (int n = 0; n < num_xmm_regs; n++) {

      __ movflt(as_XMMRegister(n), Address(rsp, off*wordSize));

      off += delta;

    }

  } else if (UseSSE >= 2) {

    // Restore whole 128bit (16 bytes) XMM registers. Do this before restoring YMM and

    // ZMM because the movdqu instruction zeros the upper part of the XMM register.

    for (int n = 0; n < num_xmm_regs; n++) {

      __ movdqu(as_XMMRegister(n), Address(rsp, off*wordSize+additional_frame_bytes));

      off += delta;

    }

  }

  if (restore_vectors) {

    if (UseAVX > 2) {

      // Restore upper half of ZMM registers.

      for (int n = 0; n < num_xmm_regs; n++) {

        __ vinsertf64x4_high(as_XMMRegister(n), Address(rsp, n*32));

      }

      __ addptr(rsp, zmm_bytes);

    }

    // Restore upper half of YMM registers.

    for (int n = 0; n < num_xmm_regs; n++) {

      __ vinsertf128_high(as_XMMRegister(n), Address(rsp, n*16));

    }

    __ addptr(rsp, ymm_bytes);

  }

  __ pop_FPU_state();

  __ addptr(rsp, FPU_regs_live*wordSize); // Pop FPU registers

  __ popf();

  __ popa();

  // Get the rbp, described implicitly by the frame sender code (no oopMap)

  __ pop(rbp);

}

void RegisterSaver::restore_result_registers(MacroAssembler* masm) {

  // Just restore result register. Only used by deoptimization. By

  // now any callee save register that needs to be restore to a c2

  // caller of the deoptee has been extracted into the vframeArray

  // and will be stuffed into the c2i adapter we create for later

  // restoration so only result registers need to be restored here.

  //

  __ frstor(Address(rsp, 0));      // Restore fpu state

  // Recover XMM & FPU state

  if( UseSSE == 1 ) {

    __ movflt(xmm0, Address(rsp, xmm0_off*wordSize));

  } else if( UseSSE >= 2 ) {

    __ movdbl(xmm0, Address(rsp, xmm0_off*wordSize));

  }

  __ movptr(rax, Address(rsp, rax_off*wordSize));

  __ movptr(rdx, Address(rsp, rdx_off*wordSize));

  // Pop all of the register save are off the stack except the return address

  __ addptr(rsp, return_off * wordSize);

}

// Is vector's size (in bytes) bigger than a size saved by default?

// 16 bytes XMM registers are saved by default using SSE2 movdqu instructions.

// Note, MaxVectorSize == 0 with UseSSE < 2 and vectors are not generated.

bool SharedRuntime::is_wide_vector(int size) {

  return size > 16;

}

size_t SharedRuntime::trampoline_size() {

  return 16;

}

void SharedRuntime::generate_trampoline(MacroAssembler *masm, address destination) {

  __ jump(RuntimeAddress(destination));

}

// The java_calling_convention describes stack locations as ideal slots on

// a frame with no abi restrictions. Since we must observe abi restrictions

// (like the placement of the register window) the slots must be biased by

// the following value.

static int reg2offset_in(VMReg r) {

  // Account for saved rbp, and return address

  // This should really be in_preserve_stack_slots

  return (r->reg2stack() + 2) * VMRegImpl::stack_slot_size;

}

static int reg2offset_out(VMReg r) {

  return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;

}

// ---------------------------------------------------------------------------

// Read the array of BasicTypes from a signature, and compute where the

// arguments should go.  Values in the VMRegPair regs array refer to 4-byte

// quantities.  Values less than SharedInfo::stack0 are registers, those above

// refer to 4-byte stack slots.  All stack slots are based off of the stack pointer

// as framesizes are fixed.

// VMRegImpl::stack0 refers to the first slot 0(sp).

// and VMRegImpl::stack0+1 refers to the memory word 4-byes higher.  Register

// up to RegisterImpl::number_of_registers) are the 32-bit

// integer registers.

// Pass first two oop/int args in registers ECX and EDX.

// Pass first two float/double args in registers XMM0 and XMM1.

// Doubles have precedence, so if you pass a mix of floats and doubles

// the doubles will grab the registers before the floats will.

// Note: the INPUTS in sig_bt are in units of Java argument words, which are

// either 32-bit or 64-bit depending on the build.  The OUTPUTS are in 32-bit

// units regardless of build. Of course for i486 there is no 64 bit build

// ---------------------------------------------------------------------------

// The compiled Java calling convention.

// Pass first two oop/int args in registers ECX and EDX.

// Pass first two float/double args in registers XMM0 and XMM1.

// Doubles have precedence, so if you pass a mix of floats and doubles

// the doubles will grab the registers before the floats will.

int SharedRuntime::java_calling_convention(const BasicType *sig_bt,

                                           VMRegPair *regs,

                                           int total_args_passed,

                                           int is_outgoing) {

  uint    stack = 0;          // Starting stack position for args on stack

  // Pass first two oop/int args in registers ECX and EDX.

  uint reg_arg0 = 9999;

  uint reg_arg1 = 9999;

  // Pass first two float/double args in registers XMM0 and XMM1.

  // Doubles have precedence, so if you pass a mix of floats and doubles

  // the doubles will grab the registers before the floats will.

  // CNC - TURNED OFF FOR non-SSE.

  //       On Intel we have to round all doubles (and most floats) at

  //       call sites by storing to the stack in any case.

  // UseSSE=0 ==> Don't Use ==> 9999+0

  // UseSSE=1 ==> Floats only ==> 9999+1

  // UseSSE>=2 ==> Floats or doubles ==> 9999+2

  enum { fltarg_dontuse = 9999+0, fltarg_float_only = 9999+1, fltarg_flt_dbl = 9999+2 };

  uint fargs = (UseSSE>=2) ? 2 : UseSSE;

  uint freg_arg0 = 9999+fargs;

  uint freg_arg1 = 9999+fargs;

  // Pass doubles & longs aligned on the stack.  First count stack slots for doubles

  int i;

  for( i = 0; i < total_args_passed; i++) {

    if( sig_bt[i] == T_DOUBLE ) {

      // first 2 doubles go in registers

      if( freg_arg0 == fltarg_flt_dbl ) freg_arg0 = i;

      else if( freg_arg1 == fltarg_flt_dbl ) freg_arg1 = i;

      else // Else double is passed low on the stack to be aligned.

        stack += 2;

    } else if( sig_bt[i] == T_LONG ) {

      stack += 2;

    }

  }

  int dstack = 0;             // Separate counter for placing doubles

  // Now pick where all else goes.

  for( i = 0; i < total_args_passed; i++) {

    // From the type and the argument number (count) compute the location

    switch( sig_bt[i] ) {

    case T_SHORT:

    case T_CHAR:

    case T_BYTE:

    case T_BOOLEAN:

    case T_INT:

    case T_ARRAY:

    case T_OBJECT:

    case T_ADDRESS:

      if( reg_arg0 == 9999 )  {

        reg_arg0 = i;

        regs[i].set1(rcx->as_VMReg());

      } else if( reg_arg1 == 9999 )  {

        reg_arg1 = i;

        regs[i].set1(rdx->as_VMReg());

      } else {

        regs[i].set1(VMRegImpl::stack2reg(stack++));

      }

      break;

    case T_FLOAT:

      if( freg_arg0 == fltarg_flt_dbl || freg_arg0 == fltarg_float_only ) {

        freg_arg0 = i;

        regs[i].set1(xmm0->as_VMReg());

      } else if( freg_arg1 == fltarg_flt_dbl || freg_arg1 == fltarg_float_only ) {

        freg_arg1 = i;

        regs[i].set1(xmm1->as_VMReg());

      } else {

        regs[i].set1(VMRegImpl::stack2reg(stack++));

      }

      break;

    case T_LONG:

      assert((i + 1) < total_args_passed && sig_bt[i+1] == T_VOID, "missing Half" );

      regs[i].set2(VMRegImpl::stack2reg(dstack));

      dstack += 2;

      break;

    case T_DOUBLE:

      assert((i + 1) < total_args_passed && sig_bt[i+1] == T_VOID, "missing Half" );

      if( freg_arg0 == (uint)i ) {

        regs[i].set2(xmm0->as_VMReg());

      } else if( freg_arg1 == (uint)i ) {

        regs[i].set2(xmm1->as_VMReg());

      } else {

        regs[i].set2(VMRegImpl::stack2reg(dstack));

        dstack += 2;

      }

      break;

    case T_VOID: regs[i].set_bad(); break;

      break;