/*

 * Copyright (c) 1998, 2016, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "classfile/systemDictionary.hpp"

#include "classfile/vmSymbols.hpp"

#include "code/codeCache.hpp"

#include "code/compiledIC.hpp"

#include "code/icBuffer.hpp"

#include "code/nmethod.hpp"

#include "code/pcDesc.hpp"

#include "code/scopeDesc.hpp"

#include "code/vtableStubs.hpp"

#include "compiler/compileBroker.hpp"

#include "compiler/oopMap.hpp"

#include "gc/g1/g1SATBCardTableModRefBS.hpp"

#include "gc/g1/heapRegion.hpp"

#include "gc/shared/barrierSet.hpp"

#include "gc/shared/collectedHeap.hpp"

#include "gc/shared/gcLocker.inline.hpp"

#include "interpreter/bytecode.hpp"

#include "interpreter/interpreter.hpp"

#include "interpreter/linkResolver.hpp"

#include "logging/log.hpp"

#include "memory/oopFactory.hpp"

#include "memory/resourceArea.hpp"

#include "oops/objArrayKlass.hpp"

#include "oops/oop.inline.hpp"

#include "oops/typeArrayOop.inline.hpp"

#include "opto/ad.hpp"

#include "opto/addnode.hpp"

#include "opto/callnode.hpp"

#include "opto/cfgnode.hpp"

#include "opto/graphKit.hpp"

#include "opto/machnode.hpp"

#include "opto/matcher.hpp"

#include "opto/memnode.hpp"

#include "opto/mulnode.hpp"

#include "opto/runtime.hpp"

#include "opto/subnode.hpp"

#include "runtime/atomic.inline.hpp"

#include "runtime/fprofiler.hpp"

#include "runtime/handles.inline.hpp"

#include "runtime/interfaceSupport.hpp"

#include "runtime/javaCalls.hpp"

#include "runtime/sharedRuntime.hpp"

#include "runtime/signature.hpp"

#include "runtime/threadCritical.hpp"

#include "runtime/vframe.hpp"

#include "runtime/vframeArray.hpp"

#include "runtime/vframe_hp.hpp"

#include "utilities/copy.hpp"

#include "utilities/preserveException.hpp"

// For debugging purposes:

//  To force FullGCALot inside a runtime function, add the following two lines

//

//  Universe::release_fullgc_alot_dummy();

//  MarkSweep::invoke(0, "Debugging");

//

// At command line specify the parameters: -XX:+FullGCALot -XX:FullGCALotStart=100000000

// Compiled code entry points

address OptoRuntime::_new_instance_Java                           = NULL;

address OptoRuntime::_new_array_Java                              = NULL;

address OptoRuntime::_new_array_nozero_Java                       = NULL;

address OptoRuntime::_multianewarray2_Java                        = NULL;

address OptoRuntime::_multianewarray3_Java                        = NULL;

address OptoRuntime::_multianewarray4_Java                        = NULL;

address OptoRuntime::_multianewarray5_Java                        = NULL;

address OptoRuntime::_multianewarrayN_Java                        = NULL;

address OptoRuntime::_g1_wb_pre_Java                              = NULL;

address OptoRuntime::_g1_wb_post_Java                             = NULL;

address OptoRuntime::_vtable_must_compile_Java                    = NULL;

address OptoRuntime::_complete_monitor_locking_Java               = NULL;

address OptoRuntime::_monitor_notify_Java                         = NULL;

address OptoRuntime::_monitor_notifyAll_Java                      = NULL;

address OptoRuntime::_rethrow_Java                                = NULL;

address OptoRuntime::_slow_arraycopy_Java                         = NULL;

address OptoRuntime::_register_finalizer_Java                     = NULL;

ExceptionBlob* OptoRuntime::_exception_blob;

// This should be called in an assertion at the start of OptoRuntime routines

// which are entered from compiled code (all of them)

#ifdef ASSERT

static bool check_compiled_frame(JavaThread* thread) {

  assert(thread->last_frame().is_runtime_frame(), "cannot call runtime directly from compiled code");

  RegisterMap map(thread, false);

  frame caller = thread->last_frame().sender(&map);

  assert(caller.is_compiled_frame(), "not being called from compiled like code");

  return true;

}

#endif // ASSERT

#define gen(env, var, type_func_gen, c_func, fancy_jump, pass_tls, save_arg_regs, return_pc) \

  var = generate_stub(env, type_func_gen, CAST_FROM_FN_PTR(address, c_func), #var, fancy_jump, pass_tls, save_arg_regs, return_pc); \

  if (var == NULL) { return false; }

bool OptoRuntime::generate(ciEnv* env) {

  generate_exception_blob();

  // Note: tls: Means fetching the return oop out of the thread-local storage

  //

  //   variable/name                       type-function-gen              , runtime method                  ,fncy_jp, tls,save_args,retpc

  // -------------------------------------------------------------------------------------------------------------------------------

  gen(env, _new_instance_Java              , new_instance_Type            , new_instance_C                  ,    0 , true , false, false);

  gen(env, _new_array_Java                 , new_array_Type               , new_array_C                     ,    0 , true , false, false);

  gen(env, _new_array_nozero_Java          , new_array_Type               , new_array_nozero_C              ,    0 , true , false, false);

  gen(env, _multianewarray2_Java           , multianewarray2_Type         , multianewarray2_C               ,    0 , true , false, false);

  gen(env, _multianewarray3_Java           , multianewarray3_Type         , multianewarray3_C               ,    0 , true , false, false);

  gen(env, _multianewarray4_Java           , multianewarray4_Type         , multianewarray4_C               ,    0 , true , false, false);

  gen(env, _multianewarray5_Java           , multianewarray5_Type         , multianewarray5_C               ,    0 , true , false, false);

  gen(env, _multianewarrayN_Java           , multianewarrayN_Type         , multianewarrayN_C               ,    0 , true , false, false);

  gen(env, _g1_wb_pre_Java                 , g1_wb_pre_Type               , SharedRuntime::g1_wb_pre        ,    0 , false, false, false);

  gen(env, _g1_wb_post_Java                , g1_wb_post_Type              , SharedRuntime::g1_wb_post       ,    0 , false, false, false);

  gen(env, _complete_monitor_locking_Java  , complete_monitor_enter_Type  , SharedRuntime::complete_monitor_locking_C, 0, false, false, false);

  gen(env, _monitor_notify_Java            , monitor_notify_Type          , monitor_notify_C                ,    0 , false, false, false);

  gen(env, _monitor_notifyAll_Java         , monitor_notify_Type          , monitor_notifyAll_C             ,    0 , false, false, false);

  gen(env, _rethrow_Java                   , rethrow_Type                 , rethrow_C                       ,    2 , true , false, true );

  gen(env, _slow_arraycopy_Java            , slow_arraycopy_Type          , SharedRuntime::slow_arraycopy_C ,    0 , false, false, false);

  gen(env, _register_finalizer_Java        , register_finalizer_Type      , register_finalizer              ,    0 , false, false, false);

  return true;

}

#undef gen

// Helper method to do generation of RunTimeStub's

address OptoRuntime::generate_stub( ciEnv* env,

                                    TypeFunc_generator gen, address C_function,

                                    const char *name, int is_fancy_jump,

                                    bool pass_tls,

                                    bool save_argument_registers,

                                    bool return_pc) {

  // Matching the default directive, we currently have no method to match.

  DirectiveSet* directive = DirectivesStack::getDefaultDirective(CompileBroker::compiler(CompLevel_full_optimization));

  ResourceMark rm;

  Compile C( env, gen, C_function, name, is_fancy_jump, pass_tls, save_argument_registers, return_pc, directive);

  DirectivesStack::release(directive);

  return  C.stub_entry_point();

}

const char* OptoRuntime::stub_name(address entry) {

#ifndef PRODUCT

  CodeBlob* cb = CodeCache::find_blob(entry);

  RuntimeStub* rs =(RuntimeStub *)cb;

  assert(rs != NULL && rs->is_runtime_stub(), "not a runtime stub");

  return rs->name();

#else

  // Fast implementation for product mode (maybe it should be inlined too)

  return "runtime stub";

#endif

}

//=============================================================================

// Opto compiler runtime routines

//=============================================================================

//=============================allocation======================================

// We failed the fast-path allocation.  Now we need to do a scavenge or GC

// and try allocation again.

void OptoRuntime::new_store_pre_barrier(JavaThread* thread) {

  // After any safepoint, just before going back to compiled code,

  // we inform the GC that we will be doing initializing writes to

  // this object in the future without emitting card-marks, so

  // GC may take any compensating steps.

  // NOTE: Keep this code consistent with GraphKit::store_barrier.

  oop new_obj = thread->vm_result();

  if (new_obj == NULL)  return;

  assert(Universe::heap()->can_elide_tlab_store_barriers(),

         "compiler must check this first");

  // GC may decide to give back a safer copy of new_obj.

  new_obj = Universe::heap()->new_store_pre_barrier(thread, new_obj);

  thread->set_vm_result(new_obj);

}

// object allocation

JRT_BLOCK_ENTRY(void, OptoRuntime::new_instance_C(Klass* klass, JavaThread* thread))

  JRT_BLOCK;

#ifndef PRODUCT

  SharedRuntime::_new_instance_ctr++;         // new instance requires GC

#endif

  assert(check_compiled_frame(thread), "incorrect caller");

  // These checks are cheap to make and support reflective allocation.

  int lh = klass->layout_helper();

  if (Klass::layout_helper_needs_slow_path(lh) || !InstanceKlass::cast(klass)->is_initialized()) {

    Handle holder(THREAD, klass->klass_holder()); // keep the klass alive

    klass->check_valid_for_instantiation(false, THREAD);

    if (!HAS_PENDING_EXCEPTION) {

      InstanceKlass::cast(klass)->initialize(THREAD);

    }

  }

  if (!HAS_PENDING_EXCEPTION) {

    // Scavenge and allocate an instance.

    Handle holder(THREAD, klass->klass_holder()); // keep the klass alive

    oop result = InstanceKlass::cast(klass)->allocate_instance(THREAD);

    thread->set_vm_result(result);

    // Pass oops back through thread local storage.  Our apparent type to Java

    // is that we return an oop, but we can block on exit from this routine and

    // a GC can trash the oop in C's return register.  The generated stub will

    // fetch the oop from TLS after any possible GC.

  }

  deoptimize_caller_frame(thread, HAS_PENDING_EXCEPTION);

  JRT_BLOCK_END;

  if (GraphKit::use_ReduceInitialCardMarks()) {

    // inform GC that we won't do card marks for initializing writes.

    new_store_pre_barrier(thread);

  }

JRT_END

// array allocation

JRT_BLOCK_ENTRY(void, OptoRuntime::new_array_C(Klass* array_type, int len, JavaThread *thread))

  JRT_BLOCK;

#ifndef PRODUCT

  SharedRuntime::_new_array_ctr++;            // new array requires GC

#endif

  assert(check_compiled_frame(thread), "incorrect caller");

  // Scavenge and allocate an instance.

  oop result;

  if (array_type->is_typeArray_klass()) {

    // The oopFactory likes to work with the element type.

    // (We could bypass the oopFactory, since it doesn't add much value.)

    BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type();

    result = oopFactory::new_typeArray(elem_type, len, THREAD);

  } else {

    // Although the oopFactory likes to work with the elem_type,

    // the compiler prefers the array_type, since it must already have

    // that latter value in hand for the fast path.

    Handle holder(THREAD, array_type->klass_holder()); // keep the array klass alive

    Klass* elem_type = ObjArrayKlass::cast(array_type)->element_klass();

    result = oopFactory::new_objArray(elem_type, len, THREAD);

  }

  // Pass oops back through thread local storage.  Our apparent type to Java

  // is that we return an oop, but we can block on exit from this routine and

  // a GC can trash the oop in C's return register.  The generated stub will

  // fetch the oop from TLS after any possible GC.

  deoptimize_caller_frame(thread, HAS_PENDING_EXCEPTION);

  thread->set_vm_result(result);

  JRT_BLOCK_END;

  if (GraphKit::use_ReduceInitialCardMarks()) {

    // inform GC that we won't do card marks for initializing writes.

    new_store_pre_barrier(thread);

  }

JRT_END

// array allocation without zeroing

JRT_BLOCK_ENTRY(void, OptoRuntime::new_array_nozero_C(Klass* array_type, int len, JavaThread *thread))

  JRT_BLOCK;

#ifndef PRODUCT

  SharedRuntime::_new_array_ctr++;            // new array requires GC

#endif

  assert(check_compiled_frame(thread), "incorrect caller");

  // Scavenge and allocate an instance.

  oop result;

  assert(array_type->is_typeArray_klass(), "should be called only for type array");

  // The oopFactory likes to work with the element type.

  BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type();

  result = oopFactory::new_typeArray_nozero(elem_type, len, THREAD);

  // Pass oops back through thread local storage.  Our apparent type to Java

  // is that we return an oop, but we can block on exit from this routine and

  // a GC can trash the oop in C's return register.  The generated stub will

  // fetch the oop from TLS after any possible GC.

  deoptimize_caller_frame(thread, HAS_PENDING_EXCEPTION);

  thread->set_vm_result(result);

  JRT_BLOCK_END;

  if (GraphKit::use_ReduceInitialCardMarks()) {

    // inform GC that we won't do card marks for initializing writes.

    new_store_pre_barrier(thread);

  }

  oop result = thread->vm_result();

  if ((len > 0) && (result != NULL) &&

      is_deoptimized_caller_frame(thread)) {

    // Zero array here if the caller is deoptimized.

    int size = ((typeArrayOop)result)->object_size();

    BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type();

    const size_t hs = arrayOopDesc::header_size(elem_type);

    // Align to next 8 bytes to avoid trashing arrays's length.

    const size_t aligned_hs = align_object_offset(hs);

    HeapWord* obj = (HeapWord*)result;

    if (aligned_hs > hs) {

      Copy::zero_to_words(obj+hs, aligned_hs-hs);

    }

    // Optimized zeroing.

    Copy::fill_to_aligned_words(obj+aligned_hs, size-aligned_hs);

  }

JRT_END

// Note: multianewarray for one dimension is handled inline by GraphKit::new_array.

// multianewarray for 2 dimensions

JRT_ENTRY(void, OptoRuntime::multianewarray2_C(Klass* elem_type, int len1, int len2, JavaThread *thread))

#ifndef PRODUCT

  SharedRuntime::_multi2_ctr++;                // multianewarray for 1 dimension

#endif

  assert(check_compiled_frame(thread), "incorrect caller");

  assert(elem_type->is_klass(), "not a class");

  jint dims[2];

  dims[0] = len1;

  dims[1] = len2;

  Handle holder(THREAD, elem_type->klass_holder()); // keep the klass alive

  oop obj = ArrayKlass::cast(elem_type)->multi_allocate(2, dims, THREAD);

  deoptimize_caller_frame(thread, HAS_PENDING_EXCEPTION);

  thread->set_vm_result(obj);

JRT_END

// multianewarray for 3 dimensions

JRT_ENTRY(void, OptoRuntime::multianewarray3_C(Klass* elem_type, int len1, int len2, int len3, JavaThread *thread))

#ifndef PRODUCT

  SharedRuntime::_multi3_ctr++;                // multianewarray for 1 dimension

#endif

  assert(check_compiled_frame(thread), "incorrect caller");

  assert(elem_type->is_klass(), "not a class");

  jint dims[3];

  dims[0] = len1;

  dims[1] = len2;

  dims[2] = len3;

  Handle holder(THREAD, elem_type->klass_holder()); // keep the klass alive

  oop obj = ArrayKlass::cast(elem_type)->multi_allocate(3, dims, THREAD);

  deoptimize_caller_frame(thread, HAS_PENDING_EXCEPTION);

  thread->set_vm_result(obj);

JRT_END

// multianewarray for 4 dimensions

JRT_ENTRY(void, OptoRuntime::multianewarray4_C(Klass* elem_type, int len1, int len2, int len3, int len4, JavaThread *thread))

#ifndef PRODUCT

  SharedRuntime::_multi4_ctr++;                // multianewarray for 1 dimension

#endif

  assert(check_compiled_frame(thread), "incorrect caller");

  assert(elem_type->is_klass(), "not a class");

  jint dims[4];

  dims[0] = len1;

  dims[1] = len2;

  dims[2] = len3;

  dims[3] = len4;

  Handle holder(THREAD, elem_type->klass_holder()); // keep the klass alive

  oop obj = ArrayKlass::cast(elem_type)->multi_allocate(4, dims, THREAD);

  deoptimize_caller_frame(thread, HAS_PENDING_EXCEPTION);

  thread->set_vm_result(obj);

JRT_END

// multianewarray for 5 dimensions

JRT_ENTRY(void, OptoRuntime::multianewarray5_C(Klass* elem_type, int len1, int len2, int len3, int len4, int len5, JavaThread *thread))

#ifndef PRODUCT

  SharedRuntime::_multi5_ctr++;                // multianewarray for 1 dimension

#endif

  assert(check_compiled_frame(thread), "incorrect caller");

  assert(elem_type->is_klass(), "not a class");

  jint dims[5];

  dims[0] = len1;

  dims[1] = len2;

  dims[2] = len3;

  dims[3] = len4;

  dims[4] = len5;

  Handle holder(THREAD, elem_type->klass_holder()); // keep the klass alive

  oop obj = ArrayKlass::cast(elem_type)->multi_allocate(5, dims, THREAD);

  deoptimize_caller_frame(thread, HAS_PENDING_EXCEPTION);

  thread->set_vm_result(obj);

JRT_END

JRT_ENTRY(void, OptoRuntime::multianewarrayN_C(Klass* elem_type, arrayOopDesc* dims, JavaThread *thread))

  assert(check_compiled_frame(thread), "incorrect caller");

  assert(elem_type->is_klass(), "not a class");

  assert(oop(dims)->is_typeArray(), "not an array");

  ResourceMark rm;

  jint len = dims->length();

  assert(len > 0, "Dimensions array should contain data");

  jint *j_dims = typeArrayOop(dims)->int_at_addr(0);

  jint *c_dims = NEW_RESOURCE_ARRAY(jint, len);

  Copy::conjoint_jints_atomic(j_dims, c_dims, len);

  Handle holder(THREAD, elem_type->klass_holder()); // keep the klass alive

  oop obj = ArrayKlass::cast(elem_type)->multi_allocate(len, c_dims, THREAD);

  deoptimize_caller_frame(thread, HAS_PENDING_EXCEPTION);

  thread->set_vm_result(obj);

JRT_END

JRT_BLOCK_ENTRY(void, OptoRuntime::monitor_notify_C(oopDesc* obj, JavaThread *thread))

  // Very few notify/notifyAll operations find any threads on the waitset, so

  // the dominant fast-path is to simply return.

  // Relatedly, it's critical that notify/notifyAll be fast in order to

  // reduce lock hold times.

  if (!SafepointSynchronize::is_synchronizing()) {

    if (ObjectSynchronizer::quick_notify(obj, thread, false)) {

      return;

    }

  }

  // This is the case the fast-path above isn't provisioned to handle.

  // The fast-path is designed to handle frequently arising cases in an efficient manner.

  // (The fast-path is just a degenerate variant of the slow-path).

  // Perform the dreaded state transition and pass control into the slow-path.

  JRT_BLOCK;

  Handle h_obj(THREAD, obj);

  ObjectSynchronizer::notify(h_obj, CHECK);

  JRT_BLOCK_END;

JRT_END

JRT_BLOCK_ENTRY(void, OptoRuntime::monitor_notifyAll_C(oopDesc* obj, JavaThread *thread))

  if (!SafepointSynchronize::is_synchronizing() ) {

    if (ObjectSynchronizer::quick_notify(obj, thread, true)) {

      return;

    }

  }

  // This is the case the fast-path above isn't provisioned to handle.

  // The fast-path is designed to handle frequently arising cases in an efficient manner.

  // (The fast-path is just a degenerate variant of the slow-path).

  // Perform the dreaded state transition and pass control into the slow-path.

  JRT_BLOCK;

  Handle h_obj(THREAD, obj);

  ObjectSynchronizer::notifyall(h_obj, CHECK);

  JRT_BLOCK_END;

JRT_END

const TypeFunc *OptoRuntime::new_instance_Type() {

  // create input type (domain)

  const Type **fields = TypeTuple::fields(1);

  fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // Klass to be allocated

  const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);

  // create result type (range)

  fields = TypeTuple::fields(1);

  fields[TypeFunc::Parms+0] = TypeRawPtr::NOTNULL; // Returned oop

  const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);

  return TypeFunc::make(domain, range);

}

const TypeFunc *OptoRuntime::athrow_Type() {

  // create input type (domain)

  const Type **fields = TypeTuple::fields(1);

  fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // Klass to be allocated

  const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);

  // create result type (range)

  fields = TypeTuple::fields(0);

  const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);

  return TypeFunc::make(domain, range);

}

const TypeFunc *OptoRuntime::new_array_Type() {

  // create input type (domain)

  const Type **fields = TypeTuple::fields(2);