/*

 * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#ifndef _WINDOWS

#include "alloca.h"

#endif

#include "asm/macroAssembler.hpp"

#include "asm/macroAssembler.inline.hpp"

#include "code/debugInfoRec.hpp"

#include "code/icBuffer.hpp"

#include "code/nativeInst.hpp"

#include "code/vtableStubs.hpp"

#include "gc/shared/collectedHeap.hpp"

#include "gc/shared/gcLocker.hpp"

#include "gc/shared/barrierSet.hpp"

#include "gc/shared/barrierSetAssembler.hpp"

#include "interpreter/interpreter.hpp"

#include "logging/log.hpp"

#include "memory/resourceArea.hpp"

#include "memory/universe.hpp"

#include "oops/compiledICHolder.hpp"

#include "oops/klass.inline.hpp"

#include "runtime/safepointMechanism.hpp"

#include "runtime/sharedRuntime.hpp"

#include "runtime/vframeArray.hpp"

#include "utilities/align.hpp"

#include "utilities/formatBuffer.hpp"

#include "vmreg_x86.inline.hpp"

#ifdef COMPILER1

#include "c1/c1_Runtime1.hpp"

#endif

#ifdef COMPILER2

#include "opto/runtime.hpp"

#endif

#if INCLUDE_JVMCI

#include "jvmci/jvmciJavaClasses.hpp"

#endif

#define __ masm->

const int StackAlignmentInSlots = StackAlignmentInBytes / VMRegImpl::stack_slot_size;

class SimpleRuntimeFrame {

  public:

  // Most of the runtime stubs have this simple frame layout.

  // This class exists to make the layout shared in one place.

  // Offsets are for compiler stack slots, which are jints.

  enum layout {

    // The frame sender code expects that rbp will be in the "natural" place and

    // will override any oopMap setting for it. We must therefore force the layout

    // so that it agrees with the frame sender code.

    rbp_off = frame::arg_reg_save_area_bytes/BytesPerInt,

    rbp_off2,

    return_off, return_off2,

    framesize

  };

};

class RegisterSaver {

  // Capture info about frame layout.  Layout offsets are in jint

  // units because compiler frame slots are jints.

#define XSAVE_AREA_BEGIN 160

#define XSAVE_AREA_YMM_BEGIN 576

#define XSAVE_AREA_ZMM_BEGIN 1152

#define XSAVE_AREA_UPPERBANK 1664

#define DEF_XMM_OFFS(regnum) xmm ## regnum ## _off = xmm_off + (regnum)*16/BytesPerInt, xmm ## regnum ## H_off

#define DEF_YMM_OFFS(regnum) ymm ## regnum ## _off = ymm_off + (regnum)*16/BytesPerInt, ymm ## regnum ## H_off

#define DEF_ZMM_OFFS(regnum) zmm ## regnum ## _off = zmm_off + (regnum-16)*64/BytesPerInt, zmm ## regnum ## H_off

  enum layout {

    fpu_state_off = frame::arg_reg_save_area_bytes/BytesPerInt, // fxsave save area

    xmm_off       = fpu_state_off + XSAVE_AREA_BEGIN/BytesPerInt,            // offset in fxsave save area

    DEF_XMM_OFFS(0),

    DEF_XMM_OFFS(1),

    // 2..15 are implied in range usage

    ymm_off = xmm_off + (XSAVE_AREA_YMM_BEGIN - XSAVE_AREA_BEGIN)/BytesPerInt,

    DEF_YMM_OFFS(0),

    DEF_YMM_OFFS(1),

    // 2..15 are implied in range usage

    zmm_high = xmm_off + (XSAVE_AREA_ZMM_BEGIN - XSAVE_AREA_BEGIN)/BytesPerInt,

    zmm_off = xmm_off + (XSAVE_AREA_UPPERBANK - XSAVE_AREA_BEGIN)/BytesPerInt,

    DEF_ZMM_OFFS(16),

    DEF_ZMM_OFFS(17),

    // 18..31 are implied in range usage

    fpu_state_end = fpu_state_off + ((FPUStateSizeInWords-1)*wordSize / BytesPerInt),

    fpu_stateH_end,

    r15_off, r15H_off,

    r14_off, r14H_off,

    r13_off, r13H_off,

    r12_off, r12H_off,

    r11_off, r11H_off,

    r10_off, r10H_off,

    r9_off,  r9H_off,

    r8_off,  r8H_off,

    rdi_off, rdiH_off,

    rsi_off, rsiH_off,

    ignore_off, ignoreH_off,  // extra copy of rbp

    rsp_off, rspH_off,

    rbx_off, rbxH_off,

    rdx_off, rdxH_off,

    rcx_off, rcxH_off,

    rax_off, raxH_off,

    // 16-byte stack alignment fill word: see MacroAssembler::push/pop_IU_state

    align_off, alignH_off,

    flags_off, flagsH_off,

    // The frame sender code expects that rbp will be in the "natural" place and

    // will override any oopMap setting for it. We must therefore force the layout

    // so that it agrees with the frame sender code.

    rbp_off, rbpH_off,        // copy of rbp we will restore

    return_off, returnH_off,  // slot for return address

    reg_save_size             // size in compiler stack slots

  };

 public:

  static OopMap* save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words, bool save_vectors = false);

  static void restore_live_registers(MacroAssembler* masm, bool restore_vectors = false);

  // Offsets into the register save area

  // Used by deoptimization when it is managing result register

  // values on its own

  static int rax_offset_in_bytes(void)    { return BytesPerInt * rax_off; }

  static int rdx_offset_in_bytes(void)    { return BytesPerInt * rdx_off; }

  static int rbx_offset_in_bytes(void)    { return BytesPerInt * rbx_off; }

  static int xmm0_offset_in_bytes(void)   { return BytesPerInt * xmm0_off; }

  static int return_offset_in_bytes(void) { return BytesPerInt * return_off; }

  // During deoptimization only the result registers need to be restored,

  // all the other values have already been extracted.

  static void restore_result_registers(MacroAssembler* masm);

};

OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words, bool save_vectors) {

  int off = 0;

  int num_xmm_regs = XMMRegisterImpl::number_of_registers;

  if (UseAVX < 3) {

    num_xmm_regs = num_xmm_regs/2;

  }

#if COMPILER2_OR_JVMCI

  if (save_vectors) {

    assert(UseAVX > 0, "Vectors larger than 16 byte long are supported only with AVX");

    assert(MaxVectorSize <= 64, "Only up to 64 byte long vectors are supported");

  }

#else

  assert(!save_vectors, "vectors are generated only by C2 and JVMCI");

#endif

  // Always make the frame size 16-byte aligned, both vector and non vector stacks are always allocated

  int frame_size_in_bytes = align_up(reg_save_size*BytesPerInt, num_xmm_regs);

  // OopMap frame size is in compiler stack slots (jint's) not bytes or words

  int frame_size_in_slots = frame_size_in_bytes / BytesPerInt;

  // CodeBlob frame size is in words.

  int frame_size_in_words = frame_size_in_bytes / wordSize;

  *total_frame_words = frame_size_in_words;

  // Save registers, fpu state, and flags.

  // We assume caller has already pushed the return address onto the

  // stack, so rsp is 8-byte aligned here.

  // We push rpb twice in this sequence because we want the real rbp

  // to be under the return like a normal enter.

  __ enter();          // rsp becomes 16-byte aligned here

  __ push_CPU_state(); // Push a multiple of 16 bytes

  // push cpu state handles this on EVEX enabled targets

  if (save_vectors) {

    // Save upper half of YMM registers(0..15)

    int base_addr = XSAVE_AREA_YMM_BEGIN;

    for (int n = 0; n < 16; n++) {

      __ vextractf128_high(Address(rsp, base_addr+n*16), as_XMMRegister(n));

    }

    if (VM_Version::supports_evex()) {

      // Save upper half of ZMM registers(0..15)

      base_addr = XSAVE_AREA_ZMM_BEGIN;

      for (int n = 0; n < 16; n++) {

        __ vextractf64x4_high(Address(rsp, base_addr+n*32), as_XMMRegister(n));

      }

      // Save full ZMM registers(16..num_xmm_regs)

      base_addr = XSAVE_AREA_UPPERBANK;

      off = 0;

      int vector_len = Assembler::AVX_512bit;

      for (int n = 16; n < num_xmm_regs; n++) {

        __ evmovdqul(Address(rsp, base_addr+(off++*64)), as_XMMRegister(n), vector_len);

      }

    }

  } else {

    if (VM_Version::supports_evex()) {

      // Save upper bank of ZMM registers(16..31) for double/float usage

      int base_addr = XSAVE_AREA_UPPERBANK;

      off = 0;

      for (int n = 16; n < num_xmm_regs; n++) {

        __ movsd(Address(rsp, base_addr+(off++*64)), as_XMMRegister(n));

      }

    }

  }

  __ vzeroupper();

  if (frame::arg_reg_save_area_bytes != 0) {

    // Allocate argument register save area

    __ subptr(rsp, frame::arg_reg_save_area_bytes);

  }

  // Set an oopmap for the call site.  This oopmap will map all

  // oop-registers and debug-info registers as callee-saved.  This

  // will allow deoptimization at this safepoint to find all possible

  // debug-info recordings, as well as let GC find all oops.

  OopMapSet *oop_maps = new OopMapSet();

  OopMap* map = new OopMap(frame_size_in_slots, 0);

#define STACK_OFFSET(x) VMRegImpl::stack2reg((x))

  map->set_callee_saved(STACK_OFFSET( rax_off ), rax->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( rcx_off ), rcx->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( rdx_off ), rdx->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( rbx_off ), rbx->as_VMReg());

  // rbp location is known implicitly by the frame sender code, needs no oopmap

  // and the location where rbp was saved by is ignored

  map->set_callee_saved(STACK_OFFSET( rsi_off ), rsi->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( rdi_off ), rdi->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( r8_off  ), r8->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( r9_off  ), r9->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( r10_off ), r10->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( r11_off ), r11->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( r12_off ), r12->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( r13_off ), r13->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( r14_off ), r14->as_VMReg());

  map->set_callee_saved(STACK_OFFSET( r15_off ), r15->as_VMReg());

  // For both AVX and EVEX we will use the legacy FXSAVE area for xmm0..xmm15,

  // on EVEX enabled targets, we get it included in the xsave area

  off = xmm0_off;

  int delta = xmm1_off - off;

  for (int n = 0; n < 16; n++) {

    XMMRegister xmm_name = as_XMMRegister(n);

    map->set_callee_saved(STACK_OFFSET(off), xmm_name->as_VMReg());

    off += delta;

  }

  if(UseAVX > 2) {

    // Obtain xmm16..xmm31 from the XSAVE area on EVEX enabled targets

    off = zmm16_off;

    delta = zmm17_off - off;

    for (int n = 16; n < num_xmm_regs; n++) {

      XMMRegister zmm_name = as_XMMRegister(n);

      map->set_callee_saved(STACK_OFFSET(off), zmm_name->as_VMReg());

      off += delta;

    }

  }

#if COMPILER2_OR_JVMCI

  if (save_vectors) {

    off = ymm0_off;

    int delta = ymm1_off - off;

    for (int n = 0; n < 16; n++) {

      XMMRegister ymm_name = as_XMMRegister(n);

      map->set_callee_saved(STACK_OFFSET(off), ymm_name->as_VMReg()->next(4));

      off += delta;

    }

  }

#endif // COMPILER2_OR_JVMCI

  // %%% These should all be a waste but we'll keep things as they were for now

  if (true) {

    map->set_callee_saved(STACK_OFFSET( raxH_off ), rax->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET( rcxH_off ), rcx->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET( rdxH_off ), rdx->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET( rbxH_off ), rbx->as_VMReg()->next());

    // rbp location is known implicitly by the frame sender code, needs no oopmap

    map->set_callee_saved(STACK_OFFSET( rsiH_off ), rsi->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET( rdiH_off ), rdi->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET( r8H_off  ), r8->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET( r9H_off  ), r9->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET( r10H_off ), r10->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET( r11H_off ), r11->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET( r12H_off ), r12->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET( r13H_off ), r13->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET( r14H_off ), r14->as_VMReg()->next());

    map->set_callee_saved(STACK_OFFSET( r15H_off ), r15->as_VMReg()->next());

    // For both AVX and EVEX we will use the legacy FXSAVE area for xmm0..xmm15,

    // on EVEX enabled targets, we get it included in the xsave area

    off = xmm0H_off;

    delta = xmm1H_off - off;

    for (int n = 0; n < 16; n++) {

      XMMRegister xmm_name = as_XMMRegister(n);

      map->set_callee_saved(STACK_OFFSET(off), xmm_name->as_VMReg()->next());

      off += delta;

    }

    if (UseAVX > 2) {

      // Obtain xmm16..xmm31 from the XSAVE area on EVEX enabled targets

      off = zmm16H_off;

      delta = zmm17H_off - off;

      for (int n = 16; n < num_xmm_regs; n++) {

        XMMRegister zmm_name = as_XMMRegister(n);

        map->set_callee_saved(STACK_OFFSET(off), zmm_name->as_VMReg()->next());

        off += delta;

      }

    }

  }

  return map;

}

void RegisterSaver::restore_live_registers(MacroAssembler* masm, bool restore_vectors) {

  int num_xmm_regs = XMMRegisterImpl::number_of_registers;

  if (UseAVX < 3) {

    num_xmm_regs = num_xmm_regs/2;

  }

  if (frame::arg_reg_save_area_bytes != 0) {

    // Pop arg register save area

    __ addptr(rsp, frame::arg_reg_save_area_bytes);

  }

#if COMPILER2_OR_JVMCI

  if (restore_vectors) {

    assert(UseAVX > 0, "Vectors larger than 16 byte long are supported only with AVX");

    assert(MaxVectorSize <= 64, "Only up to 64 byte long vectors are supported");

  }

#else

  assert(!restore_vectors, "vectors are generated only by C2");

#endif

  __ vzeroupper();

  // On EVEX enabled targets everything is handled in pop fpu state

  if (restore_vectors) {

    // Restore upper half of YMM registers (0..15)

    int base_addr = XSAVE_AREA_YMM_BEGIN;

    for (int n = 0; n < 16; n++) {

      __ vinsertf128_high(as_XMMRegister(n), Address(rsp, base_addr+n*16));

    }

    if (VM_Version::supports_evex()) {

      // Restore upper half of ZMM registers (0..15)

      base_addr = XSAVE_AREA_ZMM_BEGIN;

      for (int n = 0; n < 16; n++) {

        __ vinsertf64x4_high(as_XMMRegister(n), Address(rsp, base_addr+n*32));

      }

      // Restore full ZMM registers(16..num_xmm_regs)

      base_addr = XSAVE_AREA_UPPERBANK;

      int vector_len = Assembler::AVX_512bit;

      int off = 0;

      for (int n = 16; n < num_xmm_regs; n++) {

        __ evmovdqul(as_XMMRegister(n), Address(rsp, base_addr+(off++*64)), vector_len);

      }

    }

  } else {

    if (VM_Version::supports_evex()) {

      // Restore upper bank of ZMM registers(16..31) for double/float usage

      int base_addr = XSAVE_AREA_UPPERBANK;

      int off = 0;

      for (int n = 16; n < num_xmm_regs; n++) {

        __ movsd(as_XMMRegister(n), Address(rsp, base_addr+(off++*64)));

      }

    }

  }

  // Recover CPU state

  __ pop_CPU_state();

  // Get the rbp described implicitly by the calling convention (no oopMap)

  __ pop(rbp);

}

void RegisterSaver::restore_result_registers(MacroAssembler* masm) {

  // Just restore result register. Only used by deoptimization. By

  // now any callee save register that needs to be restored to a c2

  // caller of the deoptee has been extracted into the vframeArray

  // and will be stuffed into the c2i adapter we create for later

  // restoration so only result registers need to be restored here.

  // Restore fp result register

  __ movdbl(xmm0, Address(rsp, xmm0_offset_in_bytes()));

  // Restore integer result register

  __ movptr(rax, Address(rsp, rax_offset_in_bytes()));

  __ movptr(rdx, Address(rsp, rdx_offset_in_bytes()));

  // Pop all of the register save are off the stack except the return address

  __ addptr(rsp, return_offset_in_bytes());

}

// Is vector's size (in bytes) bigger than a size saved by default?

// 16 bytes XMM registers are saved by default using fxsave/fxrstor instructions.

bool SharedRuntime::is_wide_vector(int size) {

  return size > 16;

}

size_t SharedRuntime::trampoline_size() {

  return 16;

}

void SharedRuntime::generate_trampoline(MacroAssembler *masm, address destination) {

  __ jump(RuntimeAddress(destination));

}

// The java_calling_convention describes stack locations as ideal slots on

// a frame with no abi restrictions. Since we must observe abi restrictions

// (like the placement of the register window) the slots must be biased by

// the following value.

static int reg2offset_in(VMReg r) {

  // Account for saved rbp and return address

  // This should really be in_preserve_stack_slots

  return (r->reg2stack() + 4) * VMRegImpl::stack_slot_size;

}

static int reg2offset_out(VMReg r) {

  return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;

}

// ---------------------------------------------------------------------------

// Read the array of BasicTypes from a signature, and compute where the

// arguments should go.  Values in the VMRegPair regs array refer to 4-byte

// quantities.  Values less than VMRegImpl::stack0 are registers, those above

// refer to 4-byte stack slots.  All stack slots are based off of the stack pointer

// as framesizes are fixed.

// VMRegImpl::stack0 refers to the first slot 0(sp).

// and VMRegImpl::stack0+1 refers to the memory word 4-byes higher.  Register

// up to RegisterImpl::number_of_registers) are the 64-bit

// integer registers.

// Note: the INPUTS in sig_bt are in units of Java argument words, which are

// either 32-bit or 64-bit depending on the build.  The OUTPUTS are in 32-bit

// units regardless of build. Of course for i486 there is no 64 bit build

// The Java calling convention is a "shifted" version of the C ABI.

// By skipping the first C ABI register we can call non-static jni methods

// with small numbers of arguments without having to shuffle the arguments

// at all. Since we control the java ABI we ought to at least get some

// advantage out of it.

int SharedRuntime::java_calling_convention(const BasicType *sig_bt,

                                           VMRegPair *regs,

                                           int total_args_passed,

                                           int is_outgoing) {

  // Create the mapping between argument positions and

  // registers.

  static const Register INT_ArgReg[Argument::n_int_register_parameters_j] = {

    j_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4, j_rarg5

  };

  static const XMMRegister FP_ArgReg[Argument::n_float_register_parameters_j] = {

    j_farg0, j_farg1, j_farg2, j_farg3,

    j_farg4, j_farg5, j_farg6, j_farg7

  };

  uint int_args = 0;

  uint fp_args = 0;

  uint stk_args = 0; // inc by 2 each time

  for (int i = 0; i < total_args_passed; i++) {

    switch (sig_bt[i]) {

    case T_BOOLEAN:

    case T_CHAR:

    case T_BYTE:

    case T_SHORT:

    case T_INT:

      if (int_args < Argument::n_int_register_parameters_j) {

        regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());

      } else {

        regs[i].set1(VMRegImpl::stack2reg(stk_args));

        stk_args += 2;

      }

      break;

    case T_VOID:

      // halves of T_LONG or T_DOUBLE

      assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");

      regs[i].set_bad();

      break;

    case T_LONG:

      assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");

      // fall through

    case T_OBJECT:

    case T_ARRAY:

    case T_ADDRESS:

      if (int_args < Argument::n_int_register_parameters_j) {

        regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());

      } else {

        regs[i].set2(VMRegImpl::stack2reg(stk_args));

        stk_args += 2;

      }