/*

 * Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "asm/macroAssembler.hpp"

#include "compiler/disassembler.hpp"

#include "gc/shared/barrierSetAssembler.hpp"

#include "interpreter/bytecodeHistogram.hpp"

#include "interpreter/interp_masm.hpp"

#include "interpreter/interpreter.hpp"

#include "interpreter/interpreterRuntime.hpp"

#include "interpreter/templateInterpreterGenerator.hpp"

#include "interpreter/templateTable.hpp"

#include "oops/arrayOop.hpp"

#include "oops/methodData.hpp"

#include "oops/method.hpp"

#include "oops/oop.inline.hpp"

#include "prims/jvmtiExport.hpp"

#include "prims/jvmtiThreadState.hpp"

#include "runtime/arguments.hpp"

#include "runtime/deoptimization.hpp"

#include "runtime/frame.inline.hpp"

#include "runtime/sharedRuntime.hpp"

#include "runtime/stubRoutines.hpp"

#include "runtime/synchronizer.hpp"

#include "runtime/timer.hpp"

#include "runtime/vframeArray.hpp"

#include "utilities/debug.hpp"

#include "utilities/macros.hpp"

#define __ Disassembler::hook<InterpreterMacroAssembler>(__FILE__, __LINE__, _masm)->

// Size of interpreter code.  Increase if too small.  Interpreter will

// fail with a guarantee ("not enough space for interpreter generation");

// if too small.

// Run with +PrintInterpreter to get the VM to print out the size.

// Max size with JVMTI

#ifdef AMD64

int TemplateInterpreter::InterpreterCodeSize = JVMCI_ONLY(268) NOT_JVMCI(256) * 1024;

#else

int TemplateInterpreter::InterpreterCodeSize = 224 * 1024;

#endif // AMD64

// Global Register Names

static const Register rbcp     = LP64_ONLY(r13) NOT_LP64(rsi);

static const Register rlocals  = LP64_ONLY(r14) NOT_LP64(rdi);

const int method_offset = frame::interpreter_frame_method_offset * wordSize;

const int bcp_offset    = frame::interpreter_frame_bcp_offset    * wordSize;

const int locals_offset = frame::interpreter_frame_locals_offset * wordSize;

//-----------------------------------------------------------------------------

address TemplateInterpreterGenerator::generate_StackOverflowError_handler() {

  address entry = __ pc();

#ifdef ASSERT

  {

    Label L;

    __ lea(rax, Address(rbp,

                        frame::interpreter_frame_monitor_block_top_offset *

                        wordSize));

    __ cmpptr(rax, rsp); // rax = maximal rsp for current rbp (stack

                         // grows negative)

    __ jcc(Assembler::aboveEqual, L); // check if frame is complete

    __ stop ("interpreter frame not set up");

    __ bind(L);

  }

#endif // ASSERT

  // Restore bcp under the assumption that the current frame is still

  // interpreted

  __ restore_bcp();

  // expression stack must be empty before entering the VM if an

  // exception happened

  __ empty_expression_stack();

  // throw exception

  __ call_VM(noreg,

             CAST_FROM_FN_PTR(address,

                              InterpreterRuntime::throw_StackOverflowError));

  return entry;

}

address TemplateInterpreterGenerator::generate_ArrayIndexOutOfBounds_handler() {

  address entry = __ pc();

  // The expression stack must be empty before entering the VM if an

  // exception happened.

  __ empty_expression_stack();

  // Setup parameters.

  // ??? convention: expect aberrant index in register ebx/rbx.

  // Pass array to create more detailed exceptions.

  Register rarg = NOT_LP64(rax) LP64_ONLY(c_rarg1);

  __ call_VM(noreg,

             CAST_FROM_FN_PTR(address,

                              InterpreterRuntime::

                              throw_ArrayIndexOutOfBoundsException),

             rarg, rbx);

  return entry;

}

address TemplateInterpreterGenerator::generate_ClassCastException_handler() {

  address entry = __ pc();

  // object is at TOS

  Register rarg = NOT_LP64(rax) LP64_ONLY(c_rarg1);

  __ pop(rarg);

  // expression stack must be empty before entering the VM if an

  // exception happened

  __ empty_expression_stack();

  __ call_VM(noreg,

             CAST_FROM_FN_PTR(address,

                              InterpreterRuntime::

                              throw_ClassCastException),

             rarg);

  return entry;

}

address TemplateInterpreterGenerator::generate_exception_handler_common(

        const char* name, const char* message, bool pass_oop) {

  assert(!pass_oop || message == NULL, "either oop or message but not both");

  address entry = __ pc();

  Register rarg = NOT_LP64(rax) LP64_ONLY(c_rarg1);

  Register rarg2 = NOT_LP64(rbx) LP64_ONLY(c_rarg2);

  if (pass_oop) {

    // object is at TOS

    __ pop(rarg2);

  }

  // expression stack must be empty before entering the VM if an

  // exception happened

  __ empty_expression_stack();

  // setup parameters

  __ lea(rarg, ExternalAddress((address)name));

  if (pass_oop) {

    __ call_VM(rax, CAST_FROM_FN_PTR(address,

                                     InterpreterRuntime::

                                     create_klass_exception),

               rarg, rarg2);

  } else {

    __ lea(rarg2, ExternalAddress((address)message));

    __ call_VM(rax,

               CAST_FROM_FN_PTR(address, InterpreterRuntime::create_exception),

               rarg, rarg2);

  }

  // throw exception

  __ jump(ExternalAddress(Interpreter::throw_exception_entry()));

  return entry;

}

address TemplateInterpreterGenerator::generate_return_entry_for(TosState state, int step, size_t index_size) {

  address entry = __ pc();

#ifndef _LP64

#ifdef COMPILER2

  // The FPU stack is clean if UseSSE >= 2 but must be cleaned in other cases

  if ((state == ftos && UseSSE < 1) || (state == dtos && UseSSE < 2)) {

    for (int i = 1; i < 8; i++) {

        __ ffree(i);

    }

  } else if (UseSSE < 2) {

    __ empty_FPU_stack();

  }

#endif // COMPILER2

  if ((state == ftos && UseSSE < 1) || (state == dtos && UseSSE < 2)) {

    __ MacroAssembler::verify_FPU(1, "generate_return_entry_for compiled");

  } else {

    __ MacroAssembler::verify_FPU(0, "generate_return_entry_for compiled");

  }

  if (state == ftos) {

    __ MacroAssembler::verify_FPU(UseSSE >= 1 ? 0 : 1, "generate_return_entry_for in interpreter");

  } else if (state == dtos) {

    __ MacroAssembler::verify_FPU(UseSSE >= 2 ? 0 : 1, "generate_return_entry_for in interpreter");

  }

#endif // _LP64

  // Restore stack bottom in case i2c adjusted stack

  __ movptr(rsp, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));

  // and NULL it as marker that esp is now tos until next java call

  __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);

  __ restore_bcp();

  __ restore_locals();

  if (state == atos) {

    Register mdp = rbx;

    Register tmp = rcx;

    __ profile_return_type(mdp, rax, tmp);

  }

  const Register cache = rbx;

  const Register index = rcx;

  __ get_cache_and_index_at_bcp(cache, index, 1, index_size);

  const Register flags = cache;

  __ movl(flags, Address(cache, index, Address::times_ptr, ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::flags_offset()));

  __ andl(flags, ConstantPoolCacheEntry::parameter_size_mask);

  __ lea(rsp, Address(rsp, flags, Interpreter::stackElementScale()));

   const Register java_thread = NOT_LP64(rcx) LP64_ONLY(r15_thread);

   if (JvmtiExport::can_pop_frame()) {

     NOT_LP64(__ get_thread(java_thread));

     __ check_and_handle_popframe(java_thread);

   }

   if (JvmtiExport::can_force_early_return()) {

     NOT_LP64(__ get_thread(java_thread));

     __ check_and_handle_earlyret(java_thread);

   }

  __ dispatch_next(state, step);

  return entry;

}

address TemplateInterpreterGenerator::generate_deopt_entry_for(TosState state, int step, address continuation) {

  address entry = __ pc();

#ifndef _LP64

  if (state == ftos) {

    __ MacroAssembler::verify_FPU(UseSSE >= 1 ? 0 : 1, "generate_deopt_entry_for in interpreter");

  } else if (state == dtos) {

    __ MacroAssembler::verify_FPU(UseSSE >= 2 ? 0 : 1, "generate_deopt_entry_for in interpreter");

  }

#endif // _LP64

  // NULL last_sp until next java call

  __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);

  __ restore_bcp();

  __ restore_locals();

  const Register thread = NOT_LP64(rcx) LP64_ONLY(r15_thread);

  NOT_LP64(__ get_thread(thread));

#if INCLUDE_JVMCI

  // Check if we need to take lock at entry of synchronized method.  This can

  // only occur on method entry so emit it only for vtos with step 0.

  if ((EnableJVMCI || UseAOT) && state == vtos && step == 0) {

    Label L;

    __ cmpb(Address(thread, JavaThread::pending_monitorenter_offset()), 0);

    __ jcc(Assembler::zero, L);

    // Clear flag.

    __ movb(Address(thread, JavaThread::pending_monitorenter_offset()), 0);

    // Satisfy calling convention for lock_method().

    __ get_method(rbx);

    // Take lock.

    lock_method();

    __ bind(L);

  } else {

#ifdef ASSERT

    if (EnableJVMCI) {

      Label L;

      __ cmpb(Address(r15_thread, JavaThread::pending_monitorenter_offset()), 0);

      __ jcc(Assembler::zero, L);

      __ stop("unexpected pending monitor in deopt entry");

      __ bind(L);

    }

#endif

  }

#endif

  // handle exceptions

  {

    Label L;

    __ cmpptr(Address(thread, Thread::pending_exception_offset()), (int32_t) NULL_WORD);

    __ jcc(Assembler::zero, L);

    __ call_VM(noreg,

               CAST_FROM_FN_PTR(address,

                                InterpreterRuntime::throw_pending_exception));

    __ should_not_reach_here();

    __ bind(L);

  }

  if (continuation == NULL) {

    __ dispatch_next(state, step);

  } else {

    __ jump_to_entry(continuation);

  }

  return entry;

}

address TemplateInterpreterGenerator::generate_result_handler_for(

        BasicType type) {

  address entry = __ pc();

  switch (type) {

  case T_BOOLEAN: __ c2bool(rax);            break;

#ifndef _LP64

  case T_CHAR   : __ andptr(rax, 0xFFFF);    break;

#else

  case T_CHAR   : __ movzwl(rax, rax);       break;

#endif // _LP64

  case T_BYTE   : __ sign_extend_byte(rax);  break;

  case T_SHORT  : __ sign_extend_short(rax); break;

  case T_INT    : /* nothing to do */        break;

  case T_LONG   : /* nothing to do */        break;

  case T_VOID   : /* nothing to do */        break;

#ifndef _LP64

  case T_DOUBLE :

  case T_FLOAT  :

    { const Register t = InterpreterRuntime::SignatureHandlerGenerator::temp();

      __ pop(t);                            // remove return address first

      // Must return a result for interpreter or compiler. In SSE

      // mode, results are returned in xmm0 and the FPU stack must

      // be empty.

      if (type == T_FLOAT && UseSSE >= 1) {

        // Load ST0

        __ fld_d(Address(rsp, 0));

        // Store as float and empty fpu stack

        __ fstp_s(Address(rsp, 0));

        // and reload

        __ movflt(xmm0, Address(rsp, 0));

      } else if (type == T_DOUBLE && UseSSE >= 2 ) {

        __ movdbl(xmm0, Address(rsp, 0));

      } else {

        // restore ST0

        __ fld_d(Address(rsp, 0));

      }

      // and pop the temp

      __ addptr(rsp, 2 * wordSize);

      __ push(t);                           // restore return address

    }

    break;

#else

  case T_FLOAT  : /* nothing to do */        break;

  case T_DOUBLE : /* nothing to do */        break;

#endif // _LP64

  case T_OBJECT :

    // retrieve result from frame

    __ movptr(rax, Address(rbp, frame::interpreter_frame_oop_temp_offset*wordSize));

    // and verify it

    __ verify_oop(rax);

    break;

  default       : ShouldNotReachHere();

  }

  __ ret(0);                                   // return from result handler

  return entry;

}

address TemplateInterpreterGenerator::generate_safept_entry_for(

        TosState state,

        address runtime_entry) {

  address entry = __ pc();

  __ push(state);

  __ call_VM(noreg, runtime_entry);

  __ dispatch_via(vtos, Interpreter::_normal_table.table_for(vtos));

  return entry;

}

// Helpers for commoning out cases in the various type of method entries.

//

// increment invocation count & check for overflow

//

// Note: checking for negative value instead of overflow

//       so we have a 'sticky' overflow test

//

// rbx: method

// rcx: invocation counter

//

void TemplateInterpreterGenerator::generate_counter_incr(

        Label* overflow,

        Label* profile_method,

        Label* profile_method_continue) {

  Label done;

  // Note: In tiered we increment either counters in Method* or in MDO depending if we're profiling or not.

  if (TieredCompilation) {

    int increment = InvocationCounter::count_increment;

    Label no_mdo;

    if (ProfileInterpreter) {

      // Are we profiling?

      __ movptr(rax, Address(rbx, Method::method_data_offset()));

      __ testptr(rax, rax);

      __ jccb(Assembler::zero, no_mdo);

      // Increment counter in the MDO

      const Address mdo_invocation_counter(rax, in_bytes(MethodData::invocation_counter_offset()) +

                                                in_bytes(InvocationCounter::counter_offset()));

      const Address mask(rax, in_bytes(MethodData::invoke_mask_offset()));

      __ increment_mask_and_jump(mdo_invocation_counter, increment, mask, rcx, false, Assembler::zero, overflow);

      __ jmp(done);

    }

    __ bind(no_mdo);

    // Increment counter in MethodCounters

    const Address invocation_counter(rax,

                  MethodCounters::invocation_counter_offset() +

                  InvocationCounter::counter_offset());

    __ get_method_counters(rbx, rax, done);

    const Address mask(rax, in_bytes(MethodCounters::invoke_mask_offset()));

    __ increment_mask_and_jump(invocation_counter, increment, mask, rcx,

                               false, Assembler::zero, overflow);

    __ bind(done);

  } else { // not TieredCompilation

    const Address backedge_counter(rax,

                  MethodCounters::backedge_counter_offset() +

                  InvocationCounter::counter_offset());

    const Address invocation_counter(rax,

                  MethodCounters::invocation_counter_offset() +

                  InvocationCounter::counter_offset());

    __ get_method_counters(rbx, rax, done);

    if (ProfileInterpreter) {

      __ incrementl(Address(rax,

              MethodCounters::interpreter_invocation_counter_offset()));

    }

    // Update standard invocation counters

    __ movl(rcx, invocation_counter);

    __ incrementl(rcx, InvocationCounter::count_increment);

    __ movl(invocation_counter, rcx); // save invocation count

    __ movl(rax, backedge_counter);   // load backedge counter

    __ andl(rax, InvocationCounter::count_mask_value); // mask out the status bits

    __ addl(rcx, rax);                // add both counters

    // profile_method is non-null only for interpreted method so

    // profile_method != NULL == !native_call

    if (ProfileInterpreter && profile_method != NULL) {

      // Test to see if we should create a method data oop

      __ movptr(rax, Address(rbx, Method::method_counters_offset()));

      __ cmp32(rcx, Address(rax, in_bytes(MethodCounters::interpreter_profile_limit_offset())));

      __ jcc(Assembler::less, *profile_method_continue);

      // if no method data exists, go to profile_method

      __ test_method_data_pointer(rax, *profile_method);

    }

    __ movptr(rax, Address(rbx, Method::method_counters_offset()));

    __ cmp32(rcx, Address(rax, in_bytes(MethodCounters::interpreter_invocation_limit_offset())));

    __ jcc(Assembler::aboveEqual, *overflow);

    __ bind(done);

  }

}

void TemplateInterpreterGenerator::generate_counter_overflow(Label& do_continue) {

  // Asm interpreter on entry

  // r14/rdi - locals

  // r13/rsi - bcp

  // rbx - method

  // rdx - cpool --- DOES NOT APPEAR TO BE TRUE

  // rbp - interpreter frame

  // On return (i.e. jump to entry_point) [ back to invocation of interpreter ]

  // Everything as it was on entry

  // rdx is not restored. Doesn't appear to really be set.

  // InterpreterRuntime::frequency_counter_overflow takes two

  // arguments, the first (thread) is passed by call_VM, the second

  // indicates if the counter overflow occurs at a backwards branch

  // (NULL bcp).  We pass zero for it.  The call returns the address

  // of the verified entry point for the method or NULL if the

  // compilation did not complete (either went background or bailed

  // out).

  Register rarg = NOT_LP64(rax) LP64_ONLY(c_rarg1);

  __ movl(rarg, 0);

  __ call_VM(noreg,

             CAST_FROM_FN_PTR(address,

                              InterpreterRuntime::frequency_counter_overflow),

             rarg);

  __ movptr(rbx, Address(rbp, method_offset));   // restore Method*

  // Preserve invariant that r13/r14 contain bcp/locals of sender frame

  // and jump to the interpreted entry.

  __ jmp(do_continue, relocInfo::none);

}

// See if we've got enough room on the stack for locals plus overhead below

// JavaThread::stack_overflow_limit(). If not, throw a StackOverflowError

// without going through the signal handler, i.e., reserved and yellow zones

// will not be made usable. The shadow zone must suffice to handle the

// overflow.

// The expression stack grows down incrementally, so the normal guard