/*

 * Copyright (c) 1999, 2019, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "asm/macroAssembler.hpp"

#include "asm/macroAssembler.inline.hpp"

#include "gc/shared/barrierSet.hpp"

#include "gc/shared/barrierSetAssembler.hpp"

#include "interpreter/interpreter.hpp"

#include "memory/universe.hpp"

#include "nativeInst_x86.hpp"

#include "oops/instanceOop.hpp"

#include "oops/method.hpp"

#include "oops/objArrayKlass.hpp"

#include "oops/oop.inline.hpp"

#include "prims/methodHandles.hpp"

#include "runtime/frame.inline.hpp"

#include "runtime/handles.inline.hpp"

#include "runtime/sharedRuntime.hpp"

#include "runtime/stubCodeGenerator.hpp"

#include "runtime/stubRoutines.hpp"

#include "runtime/thread.inline.hpp"

#ifdef COMPILER2

#include "opto/runtime.hpp"

#endif

// Declaration and definition of StubGenerator (no .hpp file).

// For a more detailed description of the stub routine structure

// see the comment in stubRoutines.hpp

#define __ _masm->

#define a__ ((Assembler*)_masm)->

#ifdef PRODUCT

#define BLOCK_COMMENT(str) /* nothing */

#else

#define BLOCK_COMMENT(str) __ block_comment(str)

#endif

#define BIND(label) bind(label); BLOCK_COMMENT(#label ":")

const int MXCSR_MASK  = 0xFFC0;  // Mask out any pending exceptions

const int FPU_CNTRL_WRD_MASK = 0xFFFF;

// -------------------------------------------------------------------------------------------------------------------------

// Stub Code definitions

class StubGenerator: public StubCodeGenerator {

 private:

#ifdef PRODUCT

#define inc_counter_np(counter) ((void)0)

#else

  void inc_counter_np_(int& counter) {

    __ incrementl(ExternalAddress((address)&counter));

  }

#define inc_counter_np(counter) \

  BLOCK_COMMENT("inc_counter " #counter); \

  inc_counter_np_(counter);

#endif //PRODUCT

  void inc_copy_counter_np(BasicType t) {

#ifndef PRODUCT

    switch (t) {

    case T_BYTE:    inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); return;

    case T_SHORT:   inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); return;

    case T_INT:     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); return;

    case T_LONG:    inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); return;

    case T_OBJECT:  inc_counter_np(SharedRuntime::_oop_array_copy_ctr); return;

    default:        ShouldNotReachHere();

    }

#endif //PRODUCT

  }

  //------------------------------------------------------------------------------------------------------------------------

  // Call stubs are used to call Java from C

  //

  //    [ return_from_Java     ] <--- rsp

  //    [ argument word n      ]

  //      ...

  // -N [ argument word 1      ]

  // -7 [ Possible padding for stack alignment ]

  // -6 [ Possible padding for stack alignment ]

  // -5 [ Possible padding for stack alignment ]

  // -4 [ mxcsr save           ] <--- rsp_after_call

  // -3 [ saved rbx,            ]

  // -2 [ saved rsi            ]

  // -1 [ saved rdi            ]

  //  0 [ saved rbp,            ] <--- rbp,

  //  1 [ return address       ]

  //  2 [ ptr. to call wrapper ]

  //  3 [ result               ]

  //  4 [ result_type          ]

  //  5 [ method               ]

  //  6 [ entry_point          ]

  //  7 [ parameters           ]

  //  8 [ parameter_size       ]

  //  9 [ thread               ]

  address generate_call_stub(address& return_address) {

    StubCodeMark mark(this, "StubRoutines", "call_stub");

    address start = __ pc();

    // stub code parameters / addresses

    assert(frame::entry_frame_call_wrapper_offset == 2, "adjust this code");

    bool  sse_save = false;

    const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_catch_exception()!

    const int     locals_count_in_bytes  (4*wordSize);

    const Address mxcsr_save    (rbp, -4 * wordSize);

    const Address saved_rbx     (rbp, -3 * wordSize);

    const Address saved_rsi     (rbp, -2 * wordSize);

    const Address saved_rdi     (rbp, -1 * wordSize);

    const Address result        (rbp,  3 * wordSize);

    const Address result_type   (rbp,  4 * wordSize);

    const Address method        (rbp,  5 * wordSize);

    const Address entry_point   (rbp,  6 * wordSize);

    const Address parameters    (rbp,  7 * wordSize);

    const Address parameter_size(rbp,  8 * wordSize);

    const Address thread        (rbp,  9 * wordSize); // same as in generate_catch_exception()!

    sse_save =  UseSSE > 0;

    // stub code

    __ enter();

    __ movptr(rcx, parameter_size);              // parameter counter

    __ shlptr(rcx, Interpreter::logStackElementSize); // convert parameter count to bytes

    __ addptr(rcx, locals_count_in_bytes);       // reserve space for register saves

    __ subptr(rsp, rcx);

    __ andptr(rsp, -(StackAlignmentInBytes));    // Align stack

    // save rdi, rsi, & rbx, according to C calling conventions

    __ movptr(saved_rdi, rdi);

    __ movptr(saved_rsi, rsi);

    __ movptr(saved_rbx, rbx);

    // save and initialize %mxcsr

    if (sse_save) {

      Label skip_ldmx;

      __ stmxcsr(mxcsr_save);

      __ movl(rax, mxcsr_save);

      __ andl(rax, MXCSR_MASK);    // Only check control and mask bits

      ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());

      __ cmp32(rax, mxcsr_std);

      __ jcc(Assembler::equal, skip_ldmx);

      __ ldmxcsr(mxcsr_std);

      __ bind(skip_ldmx);

    }

    // make sure the control word is correct.

    __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));

#ifdef ASSERT

    // make sure we have no pending exceptions

    { Label L;

      __ movptr(rcx, thread);

      __ cmpptr(Address(rcx, Thread::pending_exception_offset()), (int32_t)NULL_WORD);

      __ jcc(Assembler::equal, L);

      __ stop("StubRoutines::call_stub: entered with pending exception");

      __ bind(L);

    }

#endif

    // pass parameters if any

    BLOCK_COMMENT("pass parameters if any");

    Label parameters_done;

    __ movl(rcx, parameter_size);  // parameter counter

    __ testl(rcx, rcx);

    __ jcc(Assembler::zero, parameters_done);

    // parameter passing loop

    Label loop;

    // Copy Java parameters in reverse order (receiver last)

    // Note that the argument order is inverted in the process

    // source is rdx[rcx: N-1..0]

    // dest   is rsp[rbx: 0..N-1]

    __ movptr(rdx, parameters);          // parameter pointer

    __ xorptr(rbx, rbx);

    __ BIND(loop);

    // get parameter

    __ movptr(rax, Address(rdx, rcx, Interpreter::stackElementScale(), -wordSize));

    __ movptr(Address(rsp, rbx, Interpreter::stackElementScale(),

                    Interpreter::expr_offset_in_bytes(0)), rax);          // store parameter

    __ increment(rbx);

    __ decrement(rcx);

    __ jcc(Assembler::notZero, loop);

    // call Java function

    __ BIND(parameters_done);

    __ movptr(rbx, method);           // get Method*

    __ movptr(rax, entry_point);      // get entry_point

    __ mov(rsi, rsp);                 // set sender sp

    BLOCK_COMMENT("call Java function");

    __ call(rax);

    BLOCK_COMMENT("call_stub_return_address:");

    return_address = __ pc();

#ifdef COMPILER2

    {

      Label L_skip;

      if (UseSSE >= 2) {

        __ verify_FPU(0, "call_stub_return");

      } else {

        for (int i = 1; i < 8; i++) {

          __ ffree(i);

        }

        // UseSSE <= 1 so double result should be left on TOS

        __ movl(rsi, result_type);

        __ cmpl(rsi, T_DOUBLE);

        __ jcc(Assembler::equal, L_skip);

        if (UseSSE == 0) {

          // UseSSE == 0 so float result should be left on TOS

          __ cmpl(rsi, T_FLOAT);

          __ jcc(Assembler::equal, L_skip);

        }

        __ ffree(0);

      }

      __ BIND(L_skip);

    }

#endif // COMPILER2

    // store result depending on type

    // (everything that is not T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)

    __ movptr(rdi, result);

    Label is_long, is_float, is_double, exit;

    __ movl(rsi, result_type);

    __ cmpl(rsi, T_LONG);

    __ jcc(Assembler::equal, is_long);

    __ cmpl(rsi, T_FLOAT);

    __ jcc(Assembler::equal, is_float);

    __ cmpl(rsi, T_DOUBLE);

    __ jcc(Assembler::equal, is_double);

    // handle T_INT case

    __ movl(Address(rdi, 0), rax);

    __ BIND(exit);

    // check that FPU stack is empty

    __ verify_FPU(0, "generate_call_stub");

    // pop parameters

    __ lea(rsp, rsp_after_call);

    // restore %mxcsr

    if (sse_save) {

      __ ldmxcsr(mxcsr_save);

    }

    // restore rdi, rsi and rbx,

    __ movptr(rbx, saved_rbx);

    __ movptr(rsi, saved_rsi);

    __ movptr(rdi, saved_rdi);

    __ addptr(rsp, 4*wordSize);

    // return

    __ pop(rbp);

    __ ret(0);

    // handle return types different from T_INT

    __ BIND(is_long);

    __ movl(Address(rdi, 0 * wordSize), rax);

    __ movl(Address(rdi, 1 * wordSize), rdx);

    __ jmp(exit);

    __ BIND(is_float);

    // interpreter uses xmm0 for return values

    if (UseSSE >= 1) {

      __ movflt(Address(rdi, 0), xmm0);

    } else {

      __ fstp_s(Address(rdi, 0));

    }

    __ jmp(exit);

    __ BIND(is_double);

    // interpreter uses xmm0 for return values

    if (UseSSE >= 2) {

      __ movdbl(Address(rdi, 0), xmm0);

    } else {

      __ fstp_d(Address(rdi, 0));

    }

    __ jmp(exit);

    return start;

  }

  //------------------------------------------------------------------------------------------------------------------------

  // Return point for a Java call if there's an exception thrown in Java code.

  // The exception is caught and transformed into a pending exception stored in

  // JavaThread that can be tested from within the VM.

  //

  // Note: Usually the parameters are removed by the callee. In case of an exception

  //       crossing an activation frame boundary, that is not the case if the callee

  //       is compiled code => need to setup the rsp.

  //

  // rax,: exception oop

  address generate_catch_exception() {

    StubCodeMark mark(this, "StubRoutines", "catch_exception");

    const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_call_stub()!

    const Address thread        (rbp,  9 * wordSize); // same as in generate_call_stub()!

    address start = __ pc();

    // get thread directly

    __ movptr(rcx, thread);

#ifdef ASSERT

    // verify that threads correspond

    { Label L;

      __ get_thread(rbx);

      __ cmpptr(rbx, rcx);

      __ jcc(Assembler::equal, L);

      __ stop("StubRoutines::catch_exception: threads must correspond");

      __ bind(L);

    }

#endif

    // set pending exception

    __ verify_oop(rax);

    __ movptr(Address(rcx, Thread::pending_exception_offset()), rax          );

    __ lea(Address(rcx, Thread::exception_file_offset   ()),

           ExternalAddress((address)__FILE__));

    __ movl(Address(rcx, Thread::exception_line_offset   ()), __LINE__ );

    // complete return to VM

    assert(StubRoutines::_call_stub_return_address != NULL, "_call_stub_return_address must have been generated before");

    __ jump(RuntimeAddress(StubRoutines::_call_stub_return_address));

    return start;

  }

  //------------------------------------------------------------------------------------------------------------------------

  // Continuation point for runtime calls returning with a pending exception.

  // The pending exception check happened in the runtime or native call stub.

  // The pending exception in Thread is converted into a Java-level exception.

  //

  // Contract with Java-level exception handlers:

  // rax: exception

  // rdx: throwing pc

  //

  // NOTE: At entry of this stub, exception-pc must be on stack !!

  address generate_forward_exception() {

    StubCodeMark mark(this, "StubRoutines", "forward exception");

    address start = __ pc();

    const Register thread = rcx;

    // other registers used in this stub

    const Register exception_oop = rax;

    const Register handler_addr  = rbx;

    const Register exception_pc  = rdx;

    // Upon entry, the sp points to the return address returning into Java

    // (interpreted or compiled) code; i.e., the return address becomes the

    // throwing pc.

    //

    // Arguments pushed before the runtime call are still on the stack but

    // the exception handler will reset the stack pointer -> ignore them.

    // A potential result in registers can be ignored as well.

#ifdef ASSERT

    // make sure this code is only executed if there is a pending exception

    { Label L;

      __ get_thread(thread);

      __ cmpptr(Address(thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);

      __ jcc(Assembler::notEqual, L);

      __ stop("StubRoutines::forward exception: no pending exception (1)");

      __ bind(L);

    }

#endif

    // compute exception handler into rbx,

    __ get_thread(thread);

    __ movptr(exception_pc, Address(rsp, 0));

    BLOCK_COMMENT("call exception_handler_for_return_address");

    __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), thread, exception_pc);

    __ mov(handler_addr, rax);

    // setup rax & rdx, remove return address & clear pending exception

    __ get_thread(thread);

    __ pop(exception_pc);

    __ movptr(exception_oop, Address(thread, Thread::pending_exception_offset()));

    __ movptr(Address(thread, Thread::pending_exception_offset()), NULL_WORD);

#ifdef ASSERT

    // make sure exception is set

    { Label L;

      __ testptr(exception_oop, exception_oop);

      __ jcc(Assembler::notEqual, L);

      __ stop("StubRoutines::forward exception: no pending exception (2)");

      __ bind(L);

    }

#endif

    // Verify that there is really a valid exception in RAX.

    __ verify_oop(exception_oop);

    // continue at exception handler (return address removed)

    // rax: exception

    // rbx: exception handler

    // rdx: throwing pc

    __ jmp(handler_addr);

    return start;

  }

  //----------------------------------------------------------------------------------------------------

  //

  // xchg exists as far back as 8086, lock needed for MP only

  // Stack layout immediately after call:

  //

  // 0 [ret addr ] <--- rsp

  // 1 [  ex     ]

  // 2 [  dest   ]

  //

  // Result:   *dest <- ex, return (old *dest)

  //

  // Note: win32 does not currently use this code

  address generate_atomic_xchg() {

    StubCodeMark mark(this, "StubRoutines", "atomic_xchg");

    address start = __ pc();

    __ push(rdx);

    Address exchange(rsp, 2 * wordSize);

    Address dest_addr(rsp, 3 * wordSize);

    __ movl(rax, exchange);

    __ movptr(rdx, dest_addr);

    __ xchgl(rax, Address(rdx, 0));

    __ pop(rdx);

    __ ret(0);

    return start;

  }

  //----------------------------------------------------------------------------------------------------

  // Support for void verify_mxcsr()

  //

  // This routine is used with -Xcheck:jni to verify that native

  // JNI code does not return to Java code without restoring the

  // MXCSR register to our expected state.

  address generate_verify_mxcsr() {

    StubCodeMark mark(this, "StubRoutines", "verify_mxcsr");

    address start = __ pc();

    const Address mxcsr_save(rsp, 0);

    if (CheckJNICalls && UseSSE > 0 ) {

      Label ok_ret;

      ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());

      __ push(rax);

      __ subptr(rsp, wordSize);      // allocate a temp location

      __ stmxcsr(mxcsr_save);

      __ movl(rax, mxcsr_save);

      __ andl(rax, MXCSR_MASK);

      __ cmp32(rax, mxcsr_std);

      __ jcc(Assembler::equal, ok_ret);

      __ warn("MXCSR changed by native JNI code.");

      __ ldmxcsr(mxcsr_std);

      __ bind(ok_ret);

      __ addptr(rsp, wordSize);

      __ pop(rax);

    }

    __ ret(0);

    return start;

  }

  //---------------------------------------------------------------------------

  // Support for void verify_fpu_cntrl_wrd()

  //

  // This routine is used with -Xcheck:jni to verify that native

  // JNI code does not return to Java code without restoring the

  // FP control word to our expected state.

  address generate_verify_fpu_cntrl_wrd() {

    StubCodeMark mark(this, "StubRoutines", "verify_spcw");

    address start = __ pc();

    const Address fpu_cntrl_wrd_save(rsp, 0);