/*

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "asm/assembler.hpp"

#include "interpreter/bytecodeHistogram.hpp"

#include "interpreter/interp_masm.hpp"

#include "interpreter/interpreter.hpp"

#include "interpreter/interpreterRuntime.hpp"

#include "interpreter/templateInterpreterGenerator.hpp"

#include "interpreter/templateTable.hpp"

#include "oops/arrayOop.hpp"

#include "oops/methodData.hpp"

#include "oops/method.hpp"

#include "oops/oop.inline.hpp"

#include "prims/jvmtiExport.hpp"

#include "prims/jvmtiThreadState.hpp"

#include "prims/methodHandles.hpp"

#include "runtime/arguments.hpp"

#include "runtime/deoptimization.hpp"

#include "runtime/frame.inline.hpp"

#include "runtime/sharedRuntime.hpp"

#include "runtime/stubRoutines.hpp"

#include "runtime/synchronizer.hpp"

#include "runtime/timer.hpp"

#include "runtime/vframeArray.hpp"

#include "utilities/align.hpp"

#include "utilities/debug.hpp"

#include "utilities/macros.hpp"

// Size of interpreter code.  Increase if too small.  Interpreter will

// fail with a guarantee ("not enough space for interpreter generation");

// if too small.

// Run with +PrintInterpreter to get the VM to print out the size.

// Max size with JVMTI

int TemplateInterpreter::InterpreterCodeSize = 180 * 1024;

#define __ _masm->

//------------------------------------------------------------------------------------------------------------------------

address TemplateInterpreterGenerator::generate_slow_signature_handler() {

  address entry = __ pc();

  // callee-save register for saving LR, shared with generate_native_entry

  const Register Rsaved_ret_addr = AARCH64_ONLY(R21) NOT_AARCH64(Rtmp_save0);

  __ mov(Rsaved_ret_addr, LR);

  __ mov(R1, Rmethod);

  __ mov(R2, Rlocals);

  __ mov(R3, SP);

#ifdef AARCH64

  // expand expr. stack and extended SP to avoid cutting SP in call_VM

  __ mov(Rstack_top, SP);

  __ str(Rstack_top, Address(FP, frame::interpreter_frame_extended_sp_offset * wordSize));

  __ check_stack_top();

  __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::slow_signature_handler), R1, R2, R3, false);

  __ ldp(ZR,      c_rarg1, Address(SP, 2*wordSize, post_indexed));

  __ ldp(c_rarg2, c_rarg3, Address(SP, 2*wordSize, post_indexed));

  __ ldp(c_rarg4, c_rarg5, Address(SP, 2*wordSize, post_indexed));

  __ ldp(c_rarg6, c_rarg7, Address(SP, 2*wordSize, post_indexed));

  __ ldp_d(V0, V1, Address(SP, 2*wordSize, post_indexed));

  __ ldp_d(V2, V3, Address(SP, 2*wordSize, post_indexed));

  __ ldp_d(V4, V5, Address(SP, 2*wordSize, post_indexed));

  __ ldp_d(V6, V7, Address(SP, 2*wordSize, post_indexed));

#else

  // Safer to save R9 (when scratched) since callers may have been

  // written assuming R9 survives. This is suboptimal but

  // probably not important for this slow case call site.

  // Note for R9 saving: slow_signature_handler may copy register

  // arguments above the current SP (passed as R3). It is safe for

  // call_VM to use push and pop to protect additional values on the

  // stack if needed.

  __ call_VM(CAST_FROM_FN_PTR(address, InterpreterRuntime::slow_signature_handler), true /* save R9 if needed*/);

  __ add(SP, SP, wordSize);     // Skip R0

  __ pop(RegisterSet(R1, R3));  // Load arguments passed in registers

#ifdef __ABI_HARD__

  // Few alternatives to an always-load-FP-registers approach:

  // - parse method signature to detect FP arguments

  // - keep a counter/flag on a stack indicationg number of FP arguments in the method.

  // The later has been originally implemented and tested but a conditional path could

  // eliminate any gain imposed by avoiding 8 double word loads.

  __ fldmiad(SP, FloatRegisterSet(D0, 8), writeback);

#endif // __ABI_HARD__

#endif // AARCH64

  __ ret(Rsaved_ret_addr);

  return entry;

}

//

// Various method entries (that c++ and asm interpreter agree upon)

//------------------------------------------------------------------------------------------------------------------------

//

//

// Abstract method entry

// Attempt to execute abstract method. Throw exception

address TemplateInterpreterGenerator::generate_abstract_entry(void) {

  address entry_point = __ pc();

#ifdef AARCH64

  __ restore_sp_after_call(Rtemp);

  __ restore_stack_top();

#endif

  __ empty_expression_stack();

  __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_AbstractMethodError));

  DEBUG_ONLY(STOP("generate_abstract_entry");) // Should not reach here

  return entry_point;

}

address TemplateInterpreterGenerator::generate_math_entry(AbstractInterpreter::MethodKind kind) {

  if (!InlineIntrinsics) return NULL; // Generate a vanilla entry

  // TODO: ARM

  return NULL;

  address entry_point = __ pc();

  STOP("generate_math_entry");

  return entry_point;

}

address TemplateInterpreterGenerator::generate_StackOverflowError_handler() {

  address entry = __ pc();

  // Note: There should be a minimal interpreter frame set up when stack

  // overflow occurs since we check explicitly for it now.

  //

#ifdef ASSERT

  { Label L;

    __ sub(Rtemp, FP, - frame::interpreter_frame_monitor_block_top_offset * wordSize);

    __ cmp(SP, Rtemp);  // Rtemp = maximal SP for current FP,

                        //  (stack grows negative)

    __ b(L, ls); // check if frame is complete

    __ stop ("interpreter frame not set up");

    __ bind(L);

  }

#endif // ASSERT

  // Restore bcp under the assumption that the current frame is still

  // interpreted

  __ restore_bcp();

  // expression stack must be empty before entering the VM if an exception

  // happened

  __ empty_expression_stack();

  // throw exception

  __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_StackOverflowError));

  __ should_not_reach_here();

  return entry;

}

  address entry = __ pc();

  // index is in R4_ArrayIndexOutOfBounds_index

  // expression stack must be empty before entering the VM if an exception happened

  __ empty_expression_stack();

  // setup parameters

  __ mov(R2, R4_ArrayIndexOutOfBounds_index);

  __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_ArrayIndexOutOfBoundsException), R1, R2);

  __ nop(); // to avoid filling CPU pipeline with invalid instructions

  __ nop();

  __ should_not_reach_here();

  return entry;

}

address TemplateInterpreterGenerator::generate_ClassCastException_handler() {

  address entry = __ pc();

  // object is in R2_ClassCastException_obj

  // expression stack must be empty before entering the VM if an exception

  // happened

  __ empty_expression_stack();

  __ mov(R1, R2_ClassCastException_obj);

  __ call_VM(noreg,

             CAST_FROM_FN_PTR(address,

                              InterpreterRuntime::throw_ClassCastException),

             R1);

  __ should_not_reach_here();

  return entry;

}

address TemplateInterpreterGenerator::generate_exception_handler_common(const char* name, const char* message, bool pass_oop) {

  assert(!pass_oop || message == NULL, "either oop or message but not both");

  address entry = __ pc();

  InlinedString Lname(name);

  InlinedString Lmessage(message);

  if (pass_oop) {

    // object is at TOS

    __ pop_ptr(R2);

  }

  // expression stack must be empty before entering the VM if an exception happened

  __ empty_expression_stack();

  // setup parameters

  __ ldr_literal(R1, Lname);

  if (pass_oop) {

    __ call_VM(Rexception_obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::create_klass_exception), R1, R2);

  } else {

    if (message != NULL) {

      __ ldr_literal(R2, Lmessage);

    } else {

      __ mov(R2, 0);

    }

    __ call_VM(Rexception_obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::create_exception), R1, R2);

  }

  // throw exception

  __ b(Interpreter::throw_exception_entry());

  __ nop(); // to avoid filling CPU pipeline with invalid instructions

  __ nop();

  __ bind_literal(Lname);

  if (!pass_oop && (message != NULL)) {

    __ bind_literal(Lmessage);

  }

  return entry;

}

address TemplateInterpreterGenerator::generate_return_entry_for(TosState state, int step, size_t index_size) {

  address entry = __ pc();

  __ interp_verify_oop(R0_tos, state, __FILE__, __LINE__);

#ifdef AARCH64

  __ restore_sp_after_call(Rtemp);  // Restore SP to extended SP

  __ restore_stack_top();

#else

  // Restore stack bottom in case i2c adjusted stack

  __ ldr(SP, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));

  // and NULL it as marker that SP is now tos until next java call

  __ mov(Rtemp, (int)NULL_WORD);

  __ str(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));

#endif // AARCH64

  __ restore_method();

  __ restore_bcp();

  __ restore_dispatch();

  __ restore_locals();

  const Register Rcache = R2_tmp;

  const Register Rindex = R3_tmp;

  __ get_cache_and_index_at_bcp(Rcache, Rindex, 1, index_size);

  __ add(Rtemp, Rcache, AsmOperand(Rindex, lsl, LogBytesPerWord));

  __ ldrb(Rtemp, Address(Rtemp, ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::flags_offset()));

  __ check_stack_top();

  __ add(Rstack_top, Rstack_top, AsmOperand(Rtemp, lsl, Interpreter::logStackElementSize));

#ifndef AARCH64

  __ convert_retval_to_tos(state);

#endif // !AARCH64

 __ check_and_handle_popframe();

 __ check_and_handle_earlyret();

  __ dispatch_next(state, step);

  return entry;

}

address TemplateInterpreterGenerator::generate_deopt_entry_for(TosState state, int step, address continuation) {

  address entry = __ pc();

  __ interp_verify_oop(R0_tos, state, __FILE__, __LINE__);

#ifdef AARCH64

  __ restore_sp_after_call(Rtemp);  // Restore SP to extended SP

  __ restore_stack_top();

#else

  // The stack is not extended by deopt but we must NULL last_sp as this

  // entry is like a "return".

  __ mov(Rtemp, 0);

  __ str(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));

#endif // AARCH64

  __ restore_method();

  __ restore_bcp();

  __ restore_dispatch();

  __ restore_locals();

  // handle exceptions

  { Label L;

    __ ldr(Rtemp, Address(Rthread, Thread::pending_exception_offset()));

    __ cbz(Rtemp, L);

    __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_pending_exception));

    __ should_not_reach_here();

    __ bind(L);

  }

  if (continuation == NULL) {

    __ dispatch_next(state, step);

  } else {

    __ jump_to_entry(continuation);

  }

  return entry;

}

address TemplateInterpreterGenerator::generate_result_handler_for(BasicType type) {

#ifdef AARCH64

  address entry = __ pc();

  switch (type) {

    case T_BOOLEAN:

      __ tst(R0, 0xff);

      __ cset(R0, ne);

      break;

    case T_CHAR   : __ zero_extend(R0, R0, 16);  break;

    case T_BYTE   : __ sign_extend(R0, R0,  8);  break;

    case T_SHORT  : __ sign_extend(R0, R0, 16);  break;

    case T_INT    : // fall through

    case T_LONG   : // fall through

    case T_VOID   : // fall through

    case T_FLOAT  : // fall through

    case T_DOUBLE : /* nothing to do */          break;

    case T_OBJECT :

      // retrieve result from frame

      __ ldr(R0, Address(FP, frame::interpreter_frame_oop_temp_offset * wordSize));

      // and verify it

      __ verify_oop(R0);

      break;

    default       : ShouldNotReachHere();

  }

  __ ret();

  return entry;

#else

  // Result handlers are not used on 32-bit ARM

  // since the returned value is already in appropriate format.

  __ should_not_reach_here();  // to avoid empty code block

  // The result handler non-zero indicates an object is returned and this is

  // used in the native entry code.

  return type == T_OBJECT ? (address)(-1) : NULL;

#endif // AARCH64

}

address TemplateInterpreterGenerator::generate_safept_entry_for(TosState state, address runtime_entry) {

  address entry = __ pc();

  __ push(state);

  __ call_VM(noreg, runtime_entry);

  // load current bytecode

  __ ldrb(R3_bytecode, Address(Rbcp));

  __ dispatch_only_normal(vtos);

  return entry;

}

// Helpers for commoning out cases in the various type of method entries.

//

// increment invocation count & check for overflow

//

// Note: checking for negative value instead of overflow

//       so we have a 'sticky' overflow test

//

// In: Rmethod.

//

// Uses R0, R1, Rtemp.

//

void TemplateInterpreterGenerator::generate_counter_incr(Label* overflow,

                                                 Label* profile_method,

                                                 Label* profile_method_continue) {

  Label done;

  const Register Rcounters = Rtemp;

  const Address invocation_counter(Rcounters,

                MethodCounters::invocation_counter_offset() +

                InvocationCounter::counter_offset());

  // Note: In tiered we increment either counters in MethodCounters* or

  // in MDO depending if we're profiling or not.

  if (TieredCompilation) {

    int increment = InvocationCounter::count_increment;

    Label no_mdo;

    if (ProfileInterpreter) {

      // Are we profiling?

      __ ldr(R1_tmp, Address(Rmethod, Method::method_data_offset()));

      __ cbz(R1_tmp, no_mdo);

      // Increment counter in the MDO

      const Address mdo_invocation_counter(R1_tmp,

                    in_bytes(MethodData::invocation_counter_offset()) +

                    in_bytes(InvocationCounter::counter_offset()));

      const Address mask(R1_tmp, in_bytes(MethodData::invoke_mask_offset()));

      __ increment_mask_and_jump(mdo_invocation_counter, increment, mask, R0_tmp, Rtemp, eq, overflow);

      __ b(done);

    }

    __ bind(no_mdo);

    __ get_method_counters(Rmethod, Rcounters, done);

    const Address mask(Rcounters, in_bytes(MethodCounters::invoke_mask_offset()));

    __ increment_mask_and_jump(invocation_counter, increment, mask, R0_tmp, R1_tmp, eq, overflow);

    __ bind(done);

  } else { // not TieredCompilation

    const Address backedge_counter(Rcounters,

                  MethodCounters::backedge_counter_offset() +

                  InvocationCounter::counter_offset());

    const Register Ricnt = R0_tmp;  // invocation counter

    const Register Rbcnt = R1_tmp;  // backedge counter

    __ get_method_counters(Rmethod, Rcounters, done);

    if (ProfileInterpreter) {

      const Register Riic = R1_tmp;

      __ ldr_s32(Riic, Address(Rcounters, MethodCounters::interpreter_invocation_counter_offset()));

      __ add(Riic, Riic, 1);

      __ str_32(Riic, Address(Rcounters, MethodCounters::interpreter_invocation_counter_offset()));

    }

    // Update standard invocation counters

    __ ldr_u32(Ricnt, invocation_counter);

    __ ldr_u32(Rbcnt, backedge_counter);

    __ add(Ricnt, Ricnt, InvocationCounter::count_increment);

#ifdef AARCH64

    __ andr(Rbcnt, Rbcnt, (unsigned int)InvocationCounter::count_mask_value); // mask out the status bits

#else

    __ bic(Rbcnt, Rbcnt, ~InvocationCounter::count_mask_value); // mask out the status bits

#endif // AARCH64

    __ str_32(Ricnt, invocation_counter);            // save invocation count

    __ add(Ricnt, Ricnt, Rbcnt);                     // add both counters

    // profile_method is non-null only for interpreted method so

    // profile_method != NULL == !native_call

    // BytecodeInterpreter only calls for native so code is elided.

    if (ProfileInterpreter && profile_method != NULL) {

      assert(profile_method_continue != NULL, "should be non-null");

      // Test to see if we should create a method data oop

      // Reuse R1_tmp as we don't need backedge counters anymore.

      Address profile_limit(Rcounters, in_bytes(MethodCounters::interpreter_profile_limit_offset()));

      __ ldr_s32(R1_tmp, profile_limit);

      __ cmp_32(Ricnt, R1_tmp);

      __ b(*profile_method_continue, lt);

      // if no method data exists, go to profile_method

      __ test_method_data_pointer(R1_tmp, *profile_method);

    }

    Address invoke_limit(Rcounters, in_bytes(MethodCounters::interpreter_invocation_limit_offset()));

    __ ldr_s32(R1_tmp, invoke_limit);

    __ cmp_32(Ricnt, R1_tmp);

    __ b(*overflow, hs);

    __ bind(done);

  }

}

void TemplateInterpreterGenerator::generate_counter_overflow(Label& do_continue) {

  // InterpreterRuntime::frequency_counter_overflow takes one argument

  // indicating if the counter overflow occurs at a backwards branch (non-NULL bcp).

  // The call returns the address of the verified entry point for the method or NULL

  // if the compilation did not complete (either went background or bailed out).

  __ mov(R1, (int)false);

  __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::frequency_counter_overflow), R1);

  // jump to the interpreted entry.

  __ b(do_continue);

}

void TemplateInterpreterGenerator::generate_stack_overflow_check(void) {

  // Check if we've got enough room on the stack for

  //  - overhead;

  //  - locals;

  //  - expression stack.

  //

  // Registers on entry:

  //

  // R3 = number of additional locals

  // R11 = max expression stack slots (AArch64 only)

  // Rthread

  // Rmethod

  // Registers used: R0, R1, R2, Rtemp.