/*

 * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 */

#include "precompiled.hpp"

#include "classfile/symbolTable.hpp"

#include "compiler/compileBroker.hpp"

#include "jvmci/jniAccessMark.inline.hpp"

#include "jvmci/jvmciCompilerToVM.hpp"

#include "jvmci/jvmciRuntime.hpp"

#include "logging/log.hpp"

#include "memory/oopFactory.hpp"

#include "memory/universe.hpp"

#include "oops/constantPool.inline.hpp"

#include "oops/method.inline.hpp"

#include "oops/objArrayKlass.hpp"

#include "oops/oop.inline.hpp"

#include "runtime/biasedLocking.hpp"

#include "runtime/deoptimization.hpp"

#include "runtime/fieldDescriptor.inline.hpp"

#include "runtime/frame.inline.hpp"

#include "runtime/sharedRuntime.hpp"

#if INCLUDE_G1GC

#include "gc/g1/g1ThreadLocalData.hpp"

#endif // INCLUDE_G1GC

// Simple helper to see if the caller of a runtime stub which

// entered the VM has been deoptimized

static bool caller_is_deopted() {

  JavaThread* thread = JavaThread::current();

  RegisterMap reg_map(thread, false);

  frame runtime_frame = thread->last_frame();

  frame caller_frame = runtime_frame.sender(&reg_map);

  assert(caller_frame.is_compiled_frame(), "must be compiled");

  return caller_frame.is_deoptimized_frame();

}

// Stress deoptimization

static void deopt_caller() {

  if ( !caller_is_deopted()) {

    JavaThread* thread = JavaThread::current();

    RegisterMap reg_map(thread, false);

    frame runtime_frame = thread->last_frame();

    frame caller_frame = runtime_frame.sender(&reg_map);

    Deoptimization::deoptimize_frame(thread, caller_frame.id(), Deoptimization::Reason_constraint);

    assert(caller_is_deopted(), "Must be deoptimized");

  }

}

// Manages a scope for a JVMCI runtime call that attempts a heap allocation.

// If there is a pending exception upon closing the scope and the runtime

// call is of the variety where allocation failure returns NULL without an

// exception, the following action is taken:

//   1. The pending exception is cleared

//   2. NULL is written to JavaThread::_vm_result

//   3. Checks that an OutOfMemoryError is Universe::out_of_memory_error_retry().

class RetryableAllocationMark: public StackObj {

 private:

  JavaThread* _thread;

 public:

  RetryableAllocationMark(JavaThread* thread, bool activate) {

    if (activate) {

      assert(!thread->in_retryable_allocation(), "retryable allocation scope is non-reentrant");

      _thread = thread;

      _thread->set_in_retryable_allocation(true);

    } else {

      _thread = NULL;

    }

  }

  ~RetryableAllocationMark() {

    if (_thread != NULL) {

      _thread->set_in_retryable_allocation(false);

      JavaThread* THREAD = _thread;

      if (HAS_PENDING_EXCEPTION) {

        oop ex = PENDING_EXCEPTION;

        CLEAR_PENDING_EXCEPTION;

        oop retry_oome = Universe::out_of_memory_error_retry();

        if (ex->is_a(retry_oome->klass()) && retry_oome != ex) {

          ResourceMark rm;

          fatal("Unexpected exception in scope of retryable allocation: " INTPTR_FORMAT " of type %s", p2i(ex), ex->klass()->external_name());

        }

        _thread->set_vm_result(NULL);

      }

    }

  }

};

JRT_BLOCK_ENTRY(void, JVMCIRuntime::new_instance_common(JavaThread* thread, Klass* klass, bool null_on_fail))

  JRT_BLOCK;

  assert(klass->is_klass(), "not a class");

  Handle holder(THREAD, klass->klass_holder()); // keep the klass alive

  InstanceKlass* h = InstanceKlass::cast(klass);

  {

    RetryableAllocationMark ram(thread, null_on_fail);

    h->check_valid_for_instantiation(true, CHECK);

    oop obj;

    if (null_on_fail) {

      if (!h->is_initialized()) {

        // Cannot re-execute class initialization without side effects

        // so return without attempting the initialization

        return;

      }

    } else {

      // make sure klass is initialized

      h->initialize(CHECK);

    }

    // allocate instance and return via TLS

    obj = h->allocate_instance(CHECK);

    thread->set_vm_result(obj);

  }

  JRT_BLOCK_END;

  SharedRuntime::on_slowpath_allocation_exit(thread);

JRT_END

JRT_BLOCK_ENTRY(void, JVMCIRuntime::new_array_common(JavaThread* thread, Klass* array_klass, jint length, bool null_on_fail))

  JRT_BLOCK;

  // Note: no handle for klass needed since they are not used

  //       anymore after new_objArray() and no GC can happen before.

  //       (This may have to change if this code changes!)

  assert(array_klass->is_klass(), "not a class");

  oop obj;

  if (array_klass->is_typeArray_klass()) {

    BasicType elt_type = TypeArrayKlass::cast(array_klass)->element_type();

    RetryableAllocationMark ram(thread, null_on_fail);

    obj = oopFactory::new_typeArray(elt_type, length, CHECK);

  } else {

    Handle holder(THREAD, array_klass->klass_holder()); // keep the klass alive

    Klass* elem_klass = ObjArrayKlass::cast(array_klass)->element_klass();

    RetryableAllocationMark ram(thread, null_on_fail);

    obj = oopFactory::new_objArray(elem_klass, length, CHECK);

  }

  thread->set_vm_result(obj);

  // This is pretty rare but this runtime patch is stressful to deoptimization

  // if we deoptimize here so force a deopt to stress the path.

  if (DeoptimizeALot) {

    static int deopts = 0;

    // Alternate between deoptimizing and raising an error (which will also cause a deopt)

    if (deopts++ % 2 == 0) {

      if (null_on_fail) {

        return;

      } else {

        ResourceMark rm(THREAD);

        THROW(vmSymbols::java_lang_OutOfMemoryError());

      }

    } else {

      deopt_caller();

    }

  }

  JRT_BLOCK_END;

  SharedRuntime::on_slowpath_allocation_exit(thread);

JRT_END

JRT_ENTRY(void, JVMCIRuntime::new_multi_array_common(JavaThread* thread, Klass* klass, int rank, jint* dims, bool null_on_fail))

  assert(klass->is_klass(), "not a class");

  assert(rank >= 1, "rank must be nonzero");

  Handle holder(THREAD, klass->klass_holder()); // keep the klass alive

  RetryableAllocationMark ram(thread, null_on_fail);

  oop obj = ArrayKlass::cast(klass)->multi_allocate(rank, dims, CHECK);

  thread->set_vm_result(obj);

JRT_END

JRT_ENTRY(void, JVMCIRuntime::dynamic_new_array_common(JavaThread* thread, oopDesc* element_mirror, jint length, bool null_on_fail))

  RetryableAllocationMark ram(thread, null_on_fail);

  oop obj = Reflection::reflect_new_array(element_mirror, length, CHECK);

  thread->set_vm_result(obj);

JRT_END

JRT_ENTRY(void, JVMCIRuntime::dynamic_new_instance_common(JavaThread* thread, oopDesc* type_mirror, bool null_on_fail))

  InstanceKlass* klass = InstanceKlass::cast(java_lang_Class::as_Klass(type_mirror));

  if (klass == NULL) {

    ResourceMark rm(THREAD);

    THROW(vmSymbols::java_lang_InstantiationException());

  }

  RetryableAllocationMark ram(thread, null_on_fail);

  // Create new instance (the receiver)

  klass->check_valid_for_instantiation(false, CHECK);

  if (null_on_fail) {

    if (!klass->is_initialized()) {

      // Cannot re-execute class initialization without side effects

      // so return without attempting the initialization

      return;

    }

  } else {

    // Make sure klass gets initialized

    klass->initialize(CHECK);

  }

  oop obj = klass->allocate_instance(CHECK);

  thread->set_vm_result(obj);

JRT_END

extern void vm_exit(int code);

// Enter this method from compiled code handler below. This is where we transition

// to VM mode. This is done as a helper routine so that the method called directly

// from compiled code does not have to transition to VM. This allows the entry

// method to see if the nmethod that we have just looked up a handler for has

// been deoptimized while we were in the vm. This simplifies the assembly code

// cpu directories.

//

// We are entering here from exception stub (via the entry method below)

// If there is a compiled exception handler in this method, we will continue there;

// otherwise we will unwind the stack and continue at the caller of top frame method

// Note: we enter in Java using a special JRT wrapper. This wrapper allows us to

// control the area where we can allow a safepoint. After we exit the safepoint area we can

// check to see if the handler we are going to return is now in a nmethod that has

// been deoptimized. If that is the case we return the deopt blob

// unpack_with_exception entry instead. This makes life for the exception blob easier

// because making that same check and diverting is painful from assembly language.

JRT_ENTRY_NO_ASYNC(static address, exception_handler_for_pc_helper(JavaThread* thread, oopDesc* ex, address pc, CompiledMethod*& cm))

  // Reset method handle flag.

  thread->set_is_method_handle_return(false);

  Handle exception(thread, ex);

  cm = CodeCache::find_compiled(pc);

  assert(cm != NULL, "this is not a compiled method");

  // Adjust the pc as needed/

  if (cm->is_deopt_pc(pc)) {

    RegisterMap map(thread, false);

    frame exception_frame = thread->last_frame().sender(&map);

    // if the frame isn't deopted then pc must not correspond to the caller of last_frame

    assert(exception_frame.is_deoptimized_frame(), "must be deopted");

    pc = exception_frame.pc();

  }

#ifdef ASSERT

  assert(exception.not_null(), "NULL exceptions should be handled by throw_exception");

  assert(oopDesc::is_oop(exception()), "just checking");

  // Check that exception is a subclass of Throwable, otherwise we have a VerifyError

  if (!(exception->is_a(SystemDictionary::Throwable_klass()))) {

    if (ExitVMOnVerifyError) vm_exit(-1);

    ShouldNotReachHere();

  }

#endif

  // Check the stack guard pages and reenable them if necessary and there is

  // enough space on the stack to do so.  Use fast exceptions only if the guard

  // pages are enabled.

  bool guard_pages_enabled = thread->stack_guards_enabled();

  if (!guard_pages_enabled) guard_pages_enabled = thread->reguard_stack();

  if (JvmtiExport::can_post_on_exceptions()) {

    // To ensure correct notification of exception catches and throws

    // we have to deoptimize here.  If we attempted to notify the

    // catches and throws during this exception lookup it's possible

    // we could deoptimize on the way out of the VM and end back in

    // the interpreter at the throw site.  This would result in double

    // notifications since the interpreter would also notify about

    // these same catches and throws as it unwound the frame.

    RegisterMap reg_map(thread);

    frame stub_frame = thread->last_frame();

    frame caller_frame = stub_frame.sender(&reg_map);

    // We don't really want to deoptimize the nmethod itself since we

    // can actually continue in the exception handler ourselves but I

    // don't see an easy way to have the desired effect.

    Deoptimization::deoptimize_frame(thread, caller_frame.id(), Deoptimization::Reason_constraint);

    assert(caller_is_deopted(), "Must be deoptimized");

    return SharedRuntime::deopt_blob()->unpack_with_exception_in_tls();

  }

  // ExceptionCache is used only for exceptions at call sites and not for implicit exceptions

  if (guard_pages_enabled) {

    address fast_continuation = cm->handler_for_exception_and_pc(exception, pc);

    if (fast_continuation != NULL) {

      // Set flag if return address is a method handle call site.

      thread->set_is_method_handle_return(cm->is_method_handle_return(pc));

      return fast_continuation;

    }

  }

  // If the stack guard pages are enabled, check whether there is a handler in

  // the current method.  Otherwise (guard pages disabled), force an unwind and

  // skip the exception cache update (i.e., just leave continuation==NULL).

  address continuation = NULL;

  if (guard_pages_enabled) {

    // New exception handling mechanism can support inlined methods

    // with exception handlers since the mappings are from PC to PC

    // debugging support

    // tracing

    if (log_is_enabled(Info, exceptions)) {

      ResourceMark rm;

      stringStream tempst;

      assert(cm->method() != NULL, "Unexpected null method()");

      tempst.print("compiled method <%s>\n"

                   " at PC" INTPTR_FORMAT " for thread " INTPTR_FORMAT,

                   cm->method()->print_value_string(), p2i(pc), p2i(thread));

      Exceptions::log_exception(exception, tempst.as_string());

    }

    // for AbortVMOnException flag

    NOT_PRODUCT(Exceptions::debug_check_abort(exception));

    // Clear out the exception oop and pc since looking up an

    // exception handler can cause class loading, which might throw an

    // exception and those fields are expected to be clear during

    // normal bytecode execution.

    thread->clear_exception_oop_and_pc();

    bool recursive_exception = false;

    continuation = SharedRuntime::compute_compiled_exc_handler(cm, pc, exception, false, false, recursive_exception);

    // If an exception was thrown during exception dispatch, the exception oop may have changed

    thread->set_exception_oop(exception());

    thread->set_exception_pc(pc);

    // The exception cache is used only for non-implicit exceptions

    // Update the exception cache only when another exception did

    // occur during the computation of the compiled exception handler

    // (e.g., when loading the class of the catch type).

    // Checking for exception oop equality is not

    // sufficient because some exceptions are pre-allocated and reused.

    if (continuation != NULL && !recursive_exception && !SharedRuntime::deopt_blob()->contains(continuation)) {

      cm->add_handler_for_exception_and_pc(exception, pc, continuation);

    }

  }

  // Set flag if return address is a method handle call site.

  thread->set_is_method_handle_return(cm->is_method_handle_return(pc));

  if (log_is_enabled(Info, exceptions)) {

    ResourceMark rm;

    log_info(exceptions)("Thread " PTR_FORMAT " continuing at PC " PTR_FORMAT

                         " for exception thrown at PC " PTR_FORMAT,

                         p2i(thread), p2i(continuation), p2i(pc));

  }

  return continuation;

JRT_END

// Enter this method from compiled code only if there is a Java exception handler

// in the method handling the exception.

// We are entering here from exception stub. We don't do a normal VM transition here.

// We do it in a helper. This is so we can check to see if the nmethod we have just

// searched for an exception handler has been deoptimized in the meantime.

address JVMCIRuntime::exception_handler_for_pc(JavaThread* thread) {

  oop exception = thread->exception_oop();

  address pc = thread->exception_pc();

  // Still in Java mode

  DEBUG_ONLY(ResetNoHandleMark rnhm);

  CompiledMethod* cm = NULL;

  address continuation = NULL;

  {

    // Enter VM mode by calling the helper

    ResetNoHandleMark rnhm;

    continuation = exception_handler_for_pc_helper(thread, exception, pc, cm);

  }

  // Back in JAVA, use no oops DON'T safepoint

  // Now check to see if the compiled method we were called from is now deoptimized.

  // If so we must return to the deopt blob and deoptimize the nmethod

  if (cm != NULL && caller_is_deopted()) {

    continuation = SharedRuntime::deopt_blob()->unpack_with_exception_in_tls();

  }

  assert(continuation != NULL, "no handler found");

  return continuation;

}

JRT_ENTRY_NO_ASYNC(void, JVMCIRuntime::monitorenter(JavaThread* thread, oopDesc* obj, BasicLock* lock))

  IF_TRACE_jvmci_3 {

    char type[O_BUFLEN];

    obj->klass()->name()->as_C_string(type, O_BUFLEN);

    markOop mark = obj->mark();

    TRACE_jvmci_3("%s: entered locking slow case with obj=" INTPTR_FORMAT ", type=%s, mark=" INTPTR_FORMAT ", lock=" INTPTR_FORMAT, thread->name(), p2i(obj), type, p2i(mark), p2i(lock));

    tty->flush();

  }

  if (PrintBiasedLockingStatistics) {

    Atomic::inc(BiasedLocking::slow_path_entry_count_addr());

  }

  Handle h_obj(thread, obj);

  assert(oopDesc::is_oop(h_obj()), "must be NULL or an object");

  if (UseBiasedLocking) {

    // Retry fast entry if bias is revoked to avoid unnecessary inflation

    ObjectSynchronizer::fast_enter(h_obj, lock, true, CHECK);

  } else {

    if (JVMCIUseFastLocking) {

      // When using fast locking, the compiled code has already tried the fast case

      ObjectSynchronizer::slow_enter(h_obj, lock, THREAD);

    } else {

      ObjectSynchronizer::fast_enter(h_obj, lock, false, THREAD);

    }

  }

  TRACE_jvmci_3("%s: exiting locking slow with obj=" INTPTR_FORMAT, thread->name(), p2i(obj));

JRT_END

JRT_LEAF(void, JVMCIRuntime::monitorexit(JavaThread* thread, oopDesc* obj, BasicLock* lock))

  assert(thread == JavaThread::current(), "threads must correspond");

  assert(thread->last_Java_sp(), "last_Java_sp must be set");

  // monitorexit is non-blocking (leaf routine) => no exceptions can be thrown

  EXCEPTION_MARK;

#ifdef ASSERT

  if (!oopDesc::is_oop(obj)) {

    ResetNoHandleMark rhm;

    nmethod* method = thread->last_frame().cb()->as_nmethod_or_null();

    if (method != NULL) {

      tty->print_cr("ERROR in monitorexit in method %s wrong obj " INTPTR_FORMAT, method->name(), p2i(obj));

    }

    thread->print_stack_on(tty);

    assert(false, "invalid lock object pointer dected");

  }

#endif

  if (JVMCIUseFastLocking) {

    // When using fast locking, the compiled code has already tried the fast case

    ObjectSynchronizer::slow_exit(obj, lock, THREAD);

  } else {

    ObjectSynchronizer::fast_exit(obj, lock, THREAD);

  }

  IF_TRACE_jvmci_3 {

    char type[O_BUFLEN];

    obj->klass()->name()->as_C_string(type, O_BUFLEN);

    TRACE_jvmci_3("%s: exited locking slow case with obj=" INTPTR_FORMAT ", type=%s, mark=" INTPTR_FORMAT ", lock=" INTPTR_FORMAT, thread->name(), p2i(obj), type, p2i(obj->mark()), p2i(lock));

    tty->flush();

  }

JRT_END

// Object.notify() fast path, caller does slow path

JRT_LEAF(jboolean, JVMCIRuntime::object_notify(JavaThread *thread, oopDesc* obj))

  // Very few notify/notifyAll operations find any threads on the waitset, so

  // the dominant fast-path is to simply return.

  // Relatedly, it's critical that notify/notifyAll be fast in order to

  // reduce lock hold times.

  if (!SafepointSynchronize::is_synchronizing()) {

    if (ObjectSynchronizer::quick_notify(obj, thread, false)) {

      return true;

    }

  }

  return false; // caller must perform slow path

JRT_END

// Object.notifyAll() fast path, caller does slow path

JRT_LEAF(jboolean, JVMCIRuntime::object_notifyAll(JavaThread *thread, oopDesc* obj))

  if (!SafepointSynchronize::is_synchronizing() ) {

    if (ObjectSynchronizer::quick_notify(obj, thread, true)) {

      return true;

    }

  }

  return false; // caller must perform slow path

JRT_END

JRT_ENTRY(void, JVMCIRuntime::throw_and_post_jvmti_exception(JavaThread* thread, const char* exception, const char* message))

  TempNewSymbol symbol = SymbolTable::new_symbol(exception);

  SharedRuntime::throw_and_post_jvmti_exception(thread, symbol, message);

JRT_END

JRT_ENTRY(void, JVMCIRuntime::throw_klass_external_name_exception(JavaThread* thread, const char* exception, Klass* klass))

  ResourceMark rm(thread);

  TempNewSymbol symbol = SymbolTable::new_symbol(exception);

  SharedRuntime::throw_and_post_jvmti_exception(thread, symbol, klass->external_name());

JRT_END

JRT_ENTRY(void, JVMCIRuntime::throw_class_cast_exception(JavaThread* thread, const char* exception, Klass* caster_klass, Klass* target_klass))

  ResourceMark rm(thread);

  const char* message = SharedRuntime::generate_class_cast_message(caster_klass, target_klass);

  TempNewSymbol symbol = SymbolTable::new_symbol(exception);

  SharedRuntime::throw_and_post_jvmti_exception(thread, symbol, message);

JRT_END

JRT_LEAF(void, JVMCIRuntime::log_object(JavaThread* thread, oopDesc* obj, bool as_string, bool newline))

  ttyLocker ttyl;

  if (obj == NULL) {

    tty->print("NULL");

  } else if (oopDesc::is_oop_or_null(obj, true) && (!as_string || !java_lang_String::is_instance(obj))) {

    if (oopDesc::is_oop_or_null(obj, true)) {

      char buf[O_BUFLEN];

      tty->print("%s@" INTPTR_FORMAT, obj->klass()->name()->as_C_string(buf, O_BUFLEN), p2i(obj));

    } else {

      tty->print(INTPTR_FORMAT, p2i(obj));

    }

  } else {

    ResourceMark rm;

    assert(obj != NULL && java_lang_String::is_instance(obj), "must be");

    char *buf = java_lang_String::as_utf8_string(obj);

    tty->print_raw(buf);

  }

  if (newline) {