/*

 * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.

 * Copyright (c) 2014, 2019, Red Hat Inc. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "asm/macroAssembler.hpp"

#include "asm/macroAssembler.inline.hpp"

#include "gc/shared/barrierSet.hpp"

#include "gc/shared/barrierSetAssembler.hpp"

#include "interpreter/interpreter.hpp"

#include "memory/universe.hpp"

#include "nativeInst_aarch64.hpp"

#include "oops/instanceOop.hpp"

#include "oops/method.hpp"

#include "oops/objArrayKlass.hpp"

#include "oops/oop.inline.hpp"

#include "prims/methodHandles.hpp"

#include "runtime/frame.inline.hpp"

#include "runtime/handles.inline.hpp"

#include "runtime/sharedRuntime.hpp"

#include "runtime/stubCodeGenerator.hpp"

#include "runtime/stubRoutines.hpp"

#include "runtime/thread.inline.hpp"

#include "utilities/align.hpp"

#ifdef COMPILER2

#include "opto/runtime.hpp"

#endif

#if INCLUDE_ZGC

#include "gc/z/zThreadLocalData.hpp"

#endif

// Declaration and definition of StubGenerator (no .hpp file).

// For a more detailed description of the stub routine structure

// see the comment in stubRoutines.hpp

#undef __

#define __ _masm->

#define TIMES_OOP Address::sxtw(exact_log2(UseCompressedOops ? 4 : 8))

#ifdef PRODUCT

#define BLOCK_COMMENT(str) /* nothing */

#else

#define BLOCK_COMMENT(str) __ block_comment(str)

#endif

#define BIND(label) bind(label); BLOCK_COMMENT(#label ":")

// Stub Code definitions

class StubGenerator: public StubCodeGenerator {

 private:

#ifdef PRODUCT

#define inc_counter_np(counter) ((void)0)

#else

  void inc_counter_np_(int& counter) {

    __ lea(rscratch2, ExternalAddress((address)&counter));

    __ ldrw(rscratch1, Address(rscratch2));

    __ addw(rscratch1, rscratch1, 1);

    __ strw(rscratch1, Address(rscratch2));

  }

#define inc_counter_np(counter) \

  BLOCK_COMMENT("inc_counter " #counter); \

  inc_counter_np_(counter);

#endif

  // Call stubs are used to call Java from C

  //

  // Arguments:

  //    c_rarg0:   call wrapper address                   address

  //    c_rarg1:   result                                 address

  //    c_rarg2:   result type                            BasicType

  //    c_rarg3:   method                                 Method*

  //    c_rarg4:   (interpreter) entry point              address

  //    c_rarg5:   parameters                             intptr_t*

  //    c_rarg6:   parameter size (in words)              int

  //    c_rarg7:   thread                                 Thread*

  //

  // There is no return from the stub itself as any Java result

  // is written to result

  //

  // we save r30 (lr) as the return PC at the base of the frame and

  // link r29 (fp) below it as the frame pointer installing sp (r31)

  // into fp.

  //

  // we save r0-r7, which accounts for all the c arguments.

  //

  // TODO: strictly do we need to save them all? they are treated as

  // volatile by C so could we omit saving the ones we are going to

  // place in global registers (thread? method?) or those we only use

  // during setup of the Java call?

  //

  // we don't need to save r8 which C uses as an indirect result location

  // return register.

  //

  // we don't need to save r9-r15 which both C and Java treat as

  // volatile

  //

  // we don't need to save r16-18 because Java does not use them

  //

  // we save r19-r28 which Java uses as scratch registers and C

  // expects to be callee-save

  //

  // we save the bottom 64 bits of each value stored in v8-v15; it is

  // the responsibility of the caller to preserve larger values.

  //

  // so the stub frame looks like this when we enter Java code

  //

  //     [ return_from_Java     ] <--- sp

  //     [ argument word n      ]

  //      ...

  // -27 [ argument word 1      ]

  // -26 [ saved v15            ] <--- sp_after_call

  // -25 [ saved v14            ]

  // -24 [ saved v13            ]

  // -23 [ saved v12            ]

  // -22 [ saved v11            ]

  // -21 [ saved v10            ]

  // -20 [ saved v9             ]

  // -19 [ saved v8             ]

  // -18 [ saved r28            ]

  // -17 [ saved r27            ]

  // -16 [ saved r26            ]

  // -15 [ saved r25            ]

  // -14 [ saved r24            ]

  // -13 [ saved r23            ]

  // -12 [ saved r22            ]

  // -11 [ saved r21            ]

  // -10 [ saved r20            ]

  //  -9 [ saved r19            ]

  //  -8 [ call wrapper    (r0) ]

  //  -7 [ result          (r1) ]

  //  -6 [ result type     (r2) ]

  //  -5 [ method          (r3) ]

  //  -4 [ entry point     (r4) ]

  //  -3 [ parameters      (r5) ]

  //  -2 [ parameter size  (r6) ]

  //  -1 [ thread (r7)          ]

  //   0 [ saved fp       (r29) ] <--- fp == saved sp (r31)

  //   1 [ saved lr       (r30) ]

  // Call stub stack layout word offsets from fp

  enum call_stub_layout {

    sp_after_call_off = -26,

    d15_off            = -26,

    d13_off            = -24,

    d11_off            = -22,

    d9_off             = -20,

    r28_off            = -18,

    r26_off            = -16,

    r24_off            = -14,

    r22_off            = -12,

    r20_off            = -10,

    call_wrapper_off   =  -8,

    result_off         =  -7,

    result_type_off    =  -6,

    method_off         =  -5,

    entry_point_off    =  -4,

    parameter_size_off =  -2,

    thread_off         =  -1,

    fp_f               =   0,

    retaddr_off        =   1,

  };

  address generate_call_stub(address& return_address) {

    assert((int)frame::entry_frame_after_call_words == -(int)sp_after_call_off + 1 &&

           (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off,

           "adjust this code");

    StubCodeMark mark(this, "StubRoutines", "call_stub");

    address start = __ pc();

    const Address sp_after_call(rfp, sp_after_call_off * wordSize);

    const Address call_wrapper  (rfp, call_wrapper_off   * wordSize);

    const Address result        (rfp, result_off         * wordSize);

    const Address result_type   (rfp, result_type_off    * wordSize);

    const Address method        (rfp, method_off         * wordSize);

    const Address entry_point   (rfp, entry_point_off    * wordSize);

    const Address parameter_size(rfp, parameter_size_off * wordSize);

    const Address thread        (rfp, thread_off         * wordSize);

    const Address d15_save      (rfp, d15_off * wordSize);

    const Address d13_save      (rfp, d13_off * wordSize);

    const Address d11_save      (rfp, d11_off * wordSize);

    const Address d9_save       (rfp, d9_off * wordSize);

    const Address r28_save      (rfp, r28_off * wordSize);

    const Address r26_save      (rfp, r26_off * wordSize);

    const Address r24_save      (rfp, r24_off * wordSize);

    const Address r22_save      (rfp, r22_off * wordSize);

    const Address r20_save      (rfp, r20_off * wordSize);

    // stub code

    address aarch64_entry = __ pc();

    // set up frame and move sp to end of save area

    __ enter();

    __ sub(sp, rfp, -sp_after_call_off * wordSize);

    // save register parameters and Java scratch/global registers

    // n.b. we save thread even though it gets installed in

    // rthread because we want to sanity check rthread later

    __ str(c_rarg7,  thread);

    __ strw(c_rarg6, parameter_size);

    __ stp(c_rarg4, c_rarg5,  entry_point);

    __ stp(c_rarg2, c_rarg3,  result_type);

    __ stp(c_rarg0, c_rarg1,  call_wrapper);

    __ stp(r20, r19,   r20_save);

    __ stp(r22, r21,   r22_save);

    __ stp(r24, r23,   r24_save);

    __ stp(r26, r25,   r26_save);

    __ stp(r28, r27,   r28_save);

    __ stpd(v9,  v8,   d9_save);

    __ stpd(v11, v10,  d11_save);

    __ stpd(v13, v12,  d13_save);

    __ stpd(v15, v14,  d15_save);

    // install Java thread in global register now we have saved

    // whatever value it held

    __ mov(rthread, c_rarg7);

    // And method

    __ mov(rmethod, c_rarg3);

    // set up the heapbase register

    __ reinit_heapbase();

#ifdef ASSERT

    // make sure we have no pending exceptions

    {

      Label L;

      __ ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));

      __ cmp(rscratch1, (u1)NULL_WORD);

      __ br(Assembler::EQ, L);

      __ stop("StubRoutines::call_stub: entered with pending exception");

      __ BIND(L);

    }

#endif

    // pass parameters if any

    __ mov(esp, sp);

    __ sub(rscratch1, sp, c_rarg6, ext::uxtw, LogBytesPerWord); // Move SP out of the way

    __ andr(sp, rscratch1, -2 * wordSize);

    BLOCK_COMMENT("pass parameters if any");

    Label parameters_done;

    // parameter count is still in c_rarg6

    // and parameter pointer identifying param 1 is in c_rarg5

    __ cbzw(c_rarg6, parameters_done);

    address loop = __ pc();

    __ ldr(rscratch1, Address(__ post(c_rarg5, wordSize)));

    __ subsw(c_rarg6, c_rarg6, 1);

    __ push(rscratch1);

    __ br(Assembler::GT, loop);

    __ BIND(parameters_done);

    // call Java entry -- passing methdoOop, and current sp

    //      rmethod: Method*

    //      r13: sender sp

    BLOCK_COMMENT("call Java function");

    __ mov(r13, sp);

    __ blr(c_rarg4);

    // we do this here because the notify will already have been done

    // if we get to the next instruction via an exception

    //

    // n.b. adding this instruction here affects the calculation of

    // whether or not a routine returns to the call stub (used when

    // doing stack walks) since the normal test is to check the return

    // pc against the address saved below. so we may need to allow for

    // this extra instruction in the check.

    // save current address for use by exception handling code

    return_address = __ pc();

    // store result depending on type (everything that is not

    // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)

    // n.b. this assumes Java returns an integral result in r0

    // and a floating result in j_farg0

    __ ldr(j_rarg2, result);

    Label is_long, is_float, is_double, exit;

    __ ldr(j_rarg1, result_type);

    __ cmp(j_rarg1, (u1)T_OBJECT);

    __ br(Assembler::EQ, is_long);

    __ cmp(j_rarg1, (u1)T_LONG);

    __ br(Assembler::EQ, is_long);

    __ cmp(j_rarg1, (u1)T_FLOAT);

    __ br(Assembler::EQ, is_float);

    __ cmp(j_rarg1, (u1)T_DOUBLE);

    __ br(Assembler::EQ, is_double);

    // handle T_INT case

    __ strw(r0, Address(j_rarg2));

    __ BIND(exit);

    // pop parameters

    __ sub(esp, rfp, -sp_after_call_off * wordSize);

#ifdef ASSERT

    // verify that threads correspond

    {

      Label L, S;

      __ ldr(rscratch1, thread);

      __ cmp(rthread, rscratch1);

      __ br(Assembler::NE, S);

      __ get_thread(rscratch1);

      __ cmp(rthread, rscratch1);

      __ br(Assembler::EQ, L);

      __ BIND(S);

      __ stop("StubRoutines::call_stub: threads must correspond");

      __ BIND(L);

    }

#endif

    // restore callee-save registers

    __ ldpd(v15, v14,  d15_save);

    __ ldpd(v13, v12,  d13_save);

    __ ldpd(v11, v10,  d11_save);

    __ ldpd(v9,  v8,   d9_save);

    __ ldp(r28, r27,   r28_save);

    __ ldp(r26, r25,   r26_save);

    __ ldp(r24, r23,   r24_save);

    __ ldp(r22, r21,   r22_save);

    __ ldp(r20, r19,   r20_save);

    __ ldp(c_rarg0, c_rarg1,  call_wrapper);

    __ ldrw(c_rarg2, result_type);

    __ ldr(c_rarg3,  method);

    __ ldp(c_rarg4, c_rarg5,  entry_point);

    __ ldp(c_rarg6, c_rarg7,  parameter_size);

    // leave frame and return to caller

    __ leave();

    __ ret(lr);

    // handle return types different from T_INT

    __ BIND(is_long);

    __ str(r0, Address(j_rarg2, 0));

    __ br(Assembler::AL, exit);

    __ BIND(is_float);

    __ strs(j_farg0, Address(j_rarg2, 0));

    __ br(Assembler::AL, exit);

    __ BIND(is_double);

    __ strd(j_farg0, Address(j_rarg2, 0));

    __ br(Assembler::AL, exit);

    return start;

  }

  // Return point for a Java call if there's an exception thrown in

  // Java code.  The exception is caught and transformed into a

  // pending exception stored in JavaThread that can be tested from

  // within the VM.

  //

  // Note: Usually the parameters are removed by the callee. In case

  // of an exception crossing an activation frame boundary, that is

  // not the case if the callee is compiled code => need to setup the

  // rsp.

  //

  // r0: exception oop

  address generate_catch_exception() {

    StubCodeMark mark(this, "StubRoutines", "catch_exception");

    address start = __ pc();

    // same as in generate_call_stub():

    const Address sp_after_call(rfp, sp_after_call_off * wordSize);

    const Address thread        (rfp, thread_off         * wordSize);

#ifdef ASSERT

    // verify that threads correspond

    {

      Label L, S;

      __ ldr(rscratch1, thread);

      __ cmp(rthread, rscratch1);

      __ br(Assembler::NE, S);

      __ get_thread(rscratch1);

      __ cmp(rthread, rscratch1);

      __ br(Assembler::EQ, L);

      __ bind(S);

      __ stop("StubRoutines::catch_exception: threads must correspond");

      __ bind(L);

    }

#endif

    // set pending exception

    __ verify_oop(r0);

    __ str(r0, Address(rthread, Thread::pending_exception_offset()));

    __ mov(rscratch1, (address)__FILE__);

    __ str(rscratch1, Address(rthread, Thread::exception_file_offset()));

    __ movw(rscratch1, (int)__LINE__);

    __ strw(rscratch1, Address(rthread, Thread::exception_line_offset()));

    // complete return to VM

    assert(StubRoutines::_call_stub_return_address != NULL,

           "_call_stub_return_address must have been generated before");

    __ b(StubRoutines::_call_stub_return_address);

    return start;

  }

  // Continuation point for runtime calls returning with a pending

  // exception.  The pending exception check happened in the runtime

  // or native call stub.  The pending exception in Thread is

  // converted into a Java-level exception.

  //

  // Contract with Java-level exception handlers:

  // r0: exception

  // r3: throwing pc

  //

  // NOTE: At entry of this stub, exception-pc must be in LR !!

  // NOTE: this is always used as a jump target within generated code

  // so it just needs to be generated code wiht no x86 prolog

  address generate_forward_exception() {

    StubCodeMark mark(this, "StubRoutines", "forward exception");

    address start = __ pc();

    // Upon entry, LR points to the return address returning into

    // Java (interpreted or compiled) code; i.e., the return address

    // becomes the throwing pc.

    //

    // Arguments pushed before the runtime call are still on the stack

    // but the exception handler will reset the stack pointer ->

    // ignore them.  A potential result in registers can be ignored as

    // well.

#ifdef ASSERT

    // make sure this code is only executed if there is a pending exception

    {

      Label L;

      __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));

      __ cbnz(rscratch1, L);

      __ stop("StubRoutines::forward exception: no pending exception (1)");

      __ bind(L);

    }

#endif

    // compute exception handler into r19

    // call the VM to find the handler address associated with the

    // caller address. pass thread in r0 and caller pc (ret address)

    // in r1. n.b. the caller pc is in lr, unlike x86 where it is on

    // the stack.

    __ mov(c_rarg1, lr);

    // lr will be trashed by the VM call so we move it to R19

    // (callee-saved) because we also need to pass it to the handler

    // returned by this call.

    __ mov(r19, lr);

    BLOCK_COMMENT("call exception_handler_for_return_address");

    __ call_VM_leaf(CAST_FROM_FN_PTR(address,

                         SharedRuntime::exception_handler_for_return_address),

                    rthread, c_rarg1);

    // we should not really care that lr is no longer the callee

    // address. we saved the value the handler needs in r19 so we can

    // just copy it to r3. however, the C2 handler will push its own

    // frame and then calls into the VM and the VM code asserts that

    // the PC for the frame above the handler belongs to a compiled

    // Java method. So, we restore lr here to satisfy that assert.

    __ mov(lr, r19);

    // setup r0 & r3 & clear pending exception

    __ mov(r3, r19);

    __ mov(r19, r0);

    __ ldr(r0, Address(rthread, Thread::pending_exception_offset()));

    __ str(zr, Address(rthread, Thread::pending_exception_offset()));

#ifdef ASSERT

    // make sure exception is set

    {

      Label L;

      __ cbnz(r0, L);

      __ stop("StubRoutines::forward exception: no pending exception (2)");

      __ bind(L);

    }

#endif

    // continue at exception handler

    // r0: exception

    // r3: throwing pc

    // r19: exception handler

    __ verify_oop(r0);

    __ br(r19);

    return start;

  }

  // Non-destructive plausibility checks for oops