/*

 * Copyright (c) 1999, 2017, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "asm/codeBuffer.hpp"

#include "c1/c1_CodeStubs.hpp"

#include "c1/c1_Defs.hpp"

#include "c1/c1_FrameMap.hpp"

#include "c1/c1_LIRAssembler.hpp"

#include "c1/c1_MacroAssembler.hpp"

#include "c1/c1_Runtime1.hpp"

#include "classfile/systemDictionary.hpp"

#include "classfile/vmSymbols.hpp"

#include "code/codeBlob.hpp"

#include "code/compiledIC.hpp"

#include "code/pcDesc.hpp"

#include "code/scopeDesc.hpp"

#include "code/vtableStubs.hpp"

#include "compiler/disassembler.hpp"

#include "gc/shared/barrierSet.hpp"

#include "gc/shared/collectedHeap.hpp"

#include "interpreter/bytecode.hpp"

#include "interpreter/interpreter.hpp"

#include "logging/log.hpp"

#include "memory/allocation.inline.hpp"

#include "memory/oopFactory.hpp"

#include "memory/resourceArea.hpp"

#include "oops/objArrayKlass.hpp"

#include "oops/oop.inline.hpp"

#include "runtime/atomic.hpp"

#include "runtime/biasedLocking.hpp"

#include "runtime/compilationPolicy.hpp"

#include "runtime/interfaceSupport.hpp"

#include "runtime/javaCalls.hpp"

#include "runtime/sharedRuntime.hpp"

#include "runtime/threadCritical.hpp"

#include "runtime/vframe.hpp"

#include "runtime/vframeArray.hpp"

#include "runtime/vm_version.hpp"

#include "utilities/copy.hpp"

#include "utilities/events.hpp"

// Implementation of StubAssembler

StubAssembler::StubAssembler(CodeBuffer* code, const char * name, int stub_id) : C1_MacroAssembler(code) {

  _name = name;

  _must_gc_arguments = false;

  _frame_size = no_frame_size;

  _num_rt_args = 0;

  _stub_id = stub_id;

}

void StubAssembler::set_info(const char* name, bool must_gc_arguments) {

  _name = name;

  _must_gc_arguments = must_gc_arguments;

}

void StubAssembler::set_frame_size(int size) {

  if (_frame_size == no_frame_size) {

    _frame_size = size;

  }

  assert(_frame_size == size, "can't change the frame size");

}

void StubAssembler::set_num_rt_args(int args) {

  if (_num_rt_args == 0) {

    _num_rt_args = args;

  }

  assert(_num_rt_args == args, "can't change the number of args");

}

// Implementation of Runtime1

CodeBlob* Runtime1::_blobs[Runtime1::number_of_ids];

const char *Runtime1::_blob_names[] = {

  RUNTIME1_STUBS(STUB_NAME, LAST_STUB_NAME)

};

#ifndef PRODUCT

// statistics

int Runtime1::_generic_arraycopy_cnt = 0;

int Runtime1::_generic_arraycopystub_cnt = 0;

int Runtime1::_arraycopy_slowcase_cnt = 0;

int Runtime1::_arraycopy_checkcast_cnt = 0;

int Runtime1::_arraycopy_checkcast_attempt_cnt = 0;

int Runtime1::_new_type_array_slowcase_cnt = 0;

int Runtime1::_new_object_array_slowcase_cnt = 0;

int Runtime1::_new_instance_slowcase_cnt = 0;

int Runtime1::_new_multi_array_slowcase_cnt = 0;

int Runtime1::_monitorenter_slowcase_cnt = 0;

int Runtime1::_monitorexit_slowcase_cnt = 0;

int Runtime1::_patch_code_slowcase_cnt = 0;

int Runtime1::_throw_range_check_exception_count = 0;

int Runtime1::_throw_index_exception_count = 0;

int Runtime1::_throw_div0_exception_count = 0;

int Runtime1::_throw_null_pointer_exception_count = 0;

int Runtime1::_throw_class_cast_exception_count = 0;

int Runtime1::_throw_incompatible_class_change_error_count = 0;

int Runtime1::_throw_array_store_exception_count = 0;

int Runtime1::_throw_count = 0;

static int _byte_arraycopy_stub_cnt = 0;

static int _short_arraycopy_stub_cnt = 0;

static int _int_arraycopy_stub_cnt = 0;

static int _long_arraycopy_stub_cnt = 0;

static int _oop_arraycopy_stub_cnt = 0;

address Runtime1::arraycopy_count_address(BasicType type) {

  switch (type) {

  case T_BOOLEAN:

  case T_BYTE:   return (address)&_byte_arraycopy_stub_cnt;

  case T_CHAR:

  case T_SHORT:  return (address)&_short_arraycopy_stub_cnt;

  case T_FLOAT:

  case T_INT:    return (address)&_int_arraycopy_stub_cnt;

  case T_DOUBLE:

  case T_LONG:   return (address)&_long_arraycopy_stub_cnt;

  case T_ARRAY:

  case T_OBJECT: return (address)&_oop_arraycopy_stub_cnt;

  default:

    ShouldNotReachHere();

    return NULL;

  }

}

#endif

// Simple helper to see if the caller of a runtime stub which

// entered the VM has been deoptimized

static bool caller_is_deopted() {

  JavaThread* thread = JavaThread::current();

  RegisterMap reg_map(thread, false);

  frame runtime_frame = thread->last_frame();

  frame caller_frame = runtime_frame.sender(&reg_map);

  assert(caller_frame.is_compiled_frame(), "must be compiled");

  return caller_frame.is_deoptimized_frame();

}

// Stress deoptimization

static void deopt_caller() {

  if ( !caller_is_deopted()) {

    JavaThread* thread = JavaThread::current();

    RegisterMap reg_map(thread, false);

    frame runtime_frame = thread->last_frame();

    frame caller_frame = runtime_frame.sender(&reg_map);

    Deoptimization::deoptimize_frame(thread, caller_frame.id());

    assert(caller_is_deopted(), "Must be deoptimized");

  }

}

void Runtime1::generate_blob_for(BufferBlob* buffer_blob, StubID id) {

  assert(0 <= id && id < number_of_ids, "illegal stub id");

  ResourceMark rm;

  // create code buffer for code storage

  CodeBuffer code(buffer_blob);

  OopMapSet* oop_maps;

  int frame_size;

  bool must_gc_arguments;

  Compilation::setup_code_buffer(&code, 0);

  // create assembler for code generation

  StubAssembler* sasm = new StubAssembler(&code, name_for(id), id);

  // generate code for runtime stub

  oop_maps = generate_code_for(id, sasm);

  assert(oop_maps == NULL || sasm->frame_size() != no_frame_size,

         "if stub has an oop map it must have a valid frame size");

#ifdef ASSERT

  // Make sure that stubs that need oopmaps have them

  switch (id) {

    // These stubs don't need to have an oopmap

  case dtrace_object_alloc_id:

  case g1_pre_barrier_slow_id:

  case g1_post_barrier_slow_id:

  case slow_subtype_check_id:

  case fpu2long_stub_id:

  case unwind_exception_id:

  case counter_overflow_id:

#if defined(SPARC) || defined(PPC32)

  case handle_exception_nofpu_id:  // Unused on sparc

#endif

    break;

    // All other stubs should have oopmaps

  default:

    assert(oop_maps != NULL, "must have an oopmap");

  }

#endif

  // align so printing shows nop's instead of random code at the end (SimpleStubs are aligned)

  sasm->align(BytesPerWord);

  // make sure all code is in code buffer

  sasm->flush();

  frame_size = sasm->frame_size();

  must_gc_arguments = sasm->must_gc_arguments();

  // create blob - distinguish a few special cases

  CodeBlob* blob = RuntimeStub::new_runtime_stub(name_for(id),

                                                 &code,

                                                 CodeOffsets::frame_never_safe,

                                                 frame_size,

                                                 oop_maps,

                                                 must_gc_arguments);

  // install blob

  assert(blob != NULL, "blob must exist");

  _blobs[id] = blob;

}

void Runtime1::initialize(BufferBlob* blob) {

  // platform-dependent initialization

  initialize_pd();

  // generate stubs

  for (int id = 0; id < number_of_ids; id++) generate_blob_for(blob, (StubID)id);

  // printing

#ifndef PRODUCT

  if (PrintSimpleStubs) {

    ResourceMark rm;

    for (int id = 0; id < number_of_ids; id++) {

      _blobs[id]->print();

      if (_blobs[id]->oop_maps() != NULL) {

        _blobs[id]->oop_maps()->print();

      }

    }

  }

#endif

}

CodeBlob* Runtime1::blob_for(StubID id) {

  assert(0 <= id && id < number_of_ids, "illegal stub id");

  return _blobs[id];

}

const char* Runtime1::name_for(StubID id) {

  assert(0 <= id && id < number_of_ids, "illegal stub id");

  return _blob_names[id];

}

const char* Runtime1::name_for_address(address entry) {

  for (int id = 0; id < number_of_ids; id++) {

    if (entry == entry_for((StubID)id)) return name_for((StubID)id);

  }

#define FUNCTION_CASE(a, f) \

  if ((intptr_t)a == CAST_FROM_FN_PTR(intptr_t, f))  return #f

  FUNCTION_CASE(entry, os::javaTimeMillis);

  FUNCTION_CASE(entry, os::javaTimeNanos);

  FUNCTION_CASE(entry, SharedRuntime::OSR_migration_end);

  FUNCTION_CASE(entry, SharedRuntime::d2f);

  FUNCTION_CASE(entry, SharedRuntime::d2i);

  FUNCTION_CASE(entry, SharedRuntime::d2l);

  FUNCTION_CASE(entry, SharedRuntime::dcos);

  FUNCTION_CASE(entry, SharedRuntime::dexp);

  FUNCTION_CASE(entry, SharedRuntime::dlog);

  FUNCTION_CASE(entry, SharedRuntime::dlog10);

  FUNCTION_CASE(entry, SharedRuntime::dpow);

  FUNCTION_CASE(entry, SharedRuntime::drem);

  FUNCTION_CASE(entry, SharedRuntime::dsin);

  FUNCTION_CASE(entry, SharedRuntime::dtan);

  FUNCTION_CASE(entry, SharedRuntime::f2i);

  FUNCTION_CASE(entry, SharedRuntime::f2l);

  FUNCTION_CASE(entry, SharedRuntime::frem);

  FUNCTION_CASE(entry, SharedRuntime::l2d);

  FUNCTION_CASE(entry, SharedRuntime::l2f);

  FUNCTION_CASE(entry, SharedRuntime::ldiv);

  FUNCTION_CASE(entry, SharedRuntime::lmul);

  FUNCTION_CASE(entry, SharedRuntime::lrem);

  FUNCTION_CASE(entry, SharedRuntime::lrem);

  FUNCTION_CASE(entry, SharedRuntime::dtrace_method_entry);

  FUNCTION_CASE(entry, SharedRuntime::dtrace_method_exit);

  FUNCTION_CASE(entry, is_instance_of);

  FUNCTION_CASE(entry, trace_block_entry);

#ifdef TRACE_HAVE_INTRINSICS

  FUNCTION_CASE(entry, TRACE_TIME_METHOD);

#endif

  FUNCTION_CASE(entry, StubRoutines::updateBytesCRC32());

  FUNCTION_CASE(entry, StubRoutines::updateBytesCRC32C());

  FUNCTION_CASE(entry, StubRoutines::vectorizedMismatch());

  FUNCTION_CASE(entry, StubRoutines::dexp());

  FUNCTION_CASE(entry, StubRoutines::dlog());

  FUNCTION_CASE(entry, StubRoutines::dlog10());

  FUNCTION_CASE(entry, StubRoutines::dpow());

  FUNCTION_CASE(entry, StubRoutines::dsin());

  FUNCTION_CASE(entry, StubRoutines::dcos());

  FUNCTION_CASE(entry, StubRoutines::dtan());

#undef FUNCTION_CASE

  // Soft float adds more runtime names.

  return pd_name_for_address(entry);

}

JRT_ENTRY(void, Runtime1::new_instance(JavaThread* thread, Klass* klass))

  NOT_PRODUCT(_new_instance_slowcase_cnt++;)

  assert(klass->is_klass(), "not a class");

  Handle holder(THREAD, klass->klass_holder()); // keep the klass alive

  InstanceKlass* h = InstanceKlass::cast(klass);

  h->check_valid_for_instantiation(true, CHECK);

  // make sure klass is initialized

  h->initialize(CHECK);

  // allocate instance and return via TLS

  oop obj = h->allocate_instance(CHECK);

  thread->set_vm_result(obj);

JRT_END

JRT_ENTRY(void, Runtime1::new_type_array(JavaThread* thread, Klass* klass, jint length))

  NOT_PRODUCT(_new_type_array_slowcase_cnt++;)

  // Note: no handle for klass needed since they are not used

  //       anymore after new_typeArray() and no GC can happen before.

  //       (This may have to change if this code changes!)

  assert(klass->is_klass(), "not a class");

  BasicType elt_type = TypeArrayKlass::cast(klass)->element_type();

  oop obj = oopFactory::new_typeArray(elt_type, length, CHECK);

  thread->set_vm_result(obj);

  // This is pretty rare but this runtime patch is stressful to deoptimization

  // if we deoptimize here so force a deopt to stress the path.

  if (DeoptimizeALot) {

    deopt_caller();

  }

JRT_END

JRT_ENTRY(void, Runtime1::new_object_array(JavaThread* thread, Klass* array_klass, jint length))

  NOT_PRODUCT(_new_object_array_slowcase_cnt++;)

  // Note: no handle for klass needed since they are not used

  //       anymore after new_objArray() and no GC can happen before.

  //       (This may have to change if this code changes!)

  assert(array_klass->is_klass(), "not a class");

  Handle holder(THREAD, array_klass->klass_holder()); // keep the klass alive

  Klass* elem_klass = ObjArrayKlass::cast(array_klass)->element_klass();

  objArrayOop obj = oopFactory::new_objArray(elem_klass, length, CHECK);

  thread->set_vm_result(obj);

  // This is pretty rare but this runtime patch is stressful to deoptimization

  // if we deoptimize here so force a deopt to stress the path.

  if (DeoptimizeALot) {

    deopt_caller();

  }

JRT_END

JRT_ENTRY(void, Runtime1::new_multi_array(JavaThread* thread, Klass* klass, int rank, jint* dims))

  NOT_PRODUCT(_new_multi_array_slowcase_cnt++;)

  assert(klass->is_klass(), "not a class");

  assert(rank >= 1, "rank must be nonzero");

  Handle holder(THREAD, klass->klass_holder()); // keep the klass alive

  oop obj = ArrayKlass::cast(klass)->multi_allocate(rank, dims, CHECK);

  thread->set_vm_result(obj);

JRT_END

JRT_ENTRY(void, Runtime1::unimplemented_entry(JavaThread* thread, StubID id))

  tty->print_cr("Runtime1::entry_for(%d) returned unimplemented entry point", id);

JRT_END

JRT_ENTRY(void, Runtime1::throw_array_store_exception(JavaThread* thread, oopDesc* obj))

  ResourceMark rm(thread);

  const char* klass_name = obj->klass()->external_name();

  SharedRuntime::throw_and_post_jvmti_exception(thread, vmSymbols::java_lang_ArrayStoreException(), klass_name);

JRT_END

// counter_overflow() is called from within C1-compiled methods. The enclosing method is the method

// associated with the top activation record. The inlinee (that is possibly included in the enclosing

// method) method oop is passed as an argument. In order to do that it is embedded in the code as

// a constant.

static nmethod* counter_overflow_helper(JavaThread* THREAD, int branch_bci, Method* m) {

  nmethod* osr_nm = NULL;

  methodHandle method(THREAD, m);

  RegisterMap map(THREAD, false);

  frame fr =  THREAD->last_frame().sender(&map);

  nmethod* nm = (nmethod*) fr.cb();

  assert(nm!= NULL && nm->is_nmethod(), "Sanity check");

  methodHandle enclosing_method(THREAD, nm->method());

  CompLevel level = (CompLevel)nm->comp_level();

  int bci = InvocationEntryBci;

  if (branch_bci != InvocationEntryBci) {

    // Compute destination bci

    address pc = method()->code_base() + branch_bci;

    Bytecodes::Code branch = Bytecodes::code_at(method(), pc);

    int offset = 0;

    switch (branch) {

      case Bytecodes::_if_icmplt: case Bytecodes::_iflt:

      case Bytecodes::_if_icmpgt: case Bytecodes::_ifgt:

      case Bytecodes::_if_icmple: case Bytecodes::_ifle:

      case Bytecodes::_if_icmpge: case Bytecodes::_ifge:

      case Bytecodes::_if_icmpeq: case Bytecodes::_if_acmpeq: case Bytecodes::_ifeq:

      case Bytecodes::_if_icmpne: case Bytecodes::_if_acmpne: case Bytecodes::_ifne:

      case Bytecodes::_ifnull: case Bytecodes::_ifnonnull: case Bytecodes::_goto:

        offset = (int16_t)Bytes::get_Java_u2(pc + 1);

        break;

      case Bytecodes::_goto_w:

        offset = Bytes::get_Java_u4(pc + 1);

        break;

      default: ;

    }

    bci = branch_bci + offset;

  }

  assert(!HAS_PENDING_EXCEPTION, "Should not have any exceptions pending");

  osr_nm = CompilationPolicy::policy()->event(enclosing_method, method, branch_bci, bci, level, nm, THREAD);

  assert(!HAS_PENDING_EXCEPTION, "Event handler should not throw any exceptions");

  return osr_nm;

}

JRT_BLOCK_ENTRY(address, Runtime1::counter_overflow(JavaThread* thread, int bci, Method* method))

  nmethod* osr_nm;

  JRT_BLOCK

    osr_nm = counter_overflow_helper(thread, bci, method);

    if (osr_nm != NULL) {

      RegisterMap map(thread, false);

      frame fr =  thread->last_frame().sender(&map);

      Deoptimization::deoptimize_frame(thread, fr.id());

    }

  JRT_BLOCK_END

  return NULL;

JRT_END

extern void vm_exit(int code);

// Enter this method from compiled code handler below. This is where we transition

// to VM mode. This is done as a helper routine so that the method called directly

// from compiled code does not have to transition to VM. This allows the entry

// method to see if the nmethod that we have just looked up a handler for has

// been deoptimized while we were in the vm. This simplifies the assembly code

// cpu directories.

//

// We are entering here from exception stub (via the entry method below)

// If there is a compiled exception handler in this method, we will continue there;

// otherwise we will unwind the stack and continue at the caller of top frame method

// Note: we enter in Java using a special JRT wrapper. This wrapper allows us to

// control the area where we can allow a safepoint. After we exit the safepoint area we can

// check to see if the handler we are going to return is now in a nmethod that has

// been deoptimized. If that is the case we return the deopt blob

// unpack_with_exception entry instead. This makes life for the exception blob easier

// because making that same check and diverting is painful from assembly language.

JRT_ENTRY_NO_ASYNC(static address, exception_handler_for_pc_helper(JavaThread* thread, oopDesc* ex, address pc, nmethod*& nm))

  // Reset method handle flag.

  thread->set_is_method_handle_return(false);

  Handle exception(thread, ex);

  nm = CodeCache::find_nmethod(pc);

  assert(nm != NULL, "this is not an nmethod");

  // Adjust the pc as needed/

  if (nm->is_deopt_pc(pc)) {

    RegisterMap map(thread, false);

    frame exception_frame = thread->last_frame().sender(&map);

    // if the frame isn't deopted then pc must not correspond to the caller of last_frame

    assert(exception_frame.is_deoptimized_frame(), "must be deopted");

    pc = exception_frame.pc();

  }

#ifdef ASSERT

  assert(exception.not_null(), "NULL exceptions should be handled by throw_exception");

  // Check that exception is a subclass of Throwable, otherwise we have a VerifyError

  if (!(exception->is_a(SystemDictionary::Throwable_klass()))) {

    if (ExitVMOnVerifyError) vm_exit(-1);

    ShouldNotReachHere();

  }

#endif

  // Check the stack guard pages and reenable them if necessary and there is

  // enough space on the stack to do so.  Use fast exceptions only if the guard

  // pages are enabled.

  bool guard_pages_enabled = thread->stack_guards_enabled();

  if (!guard_pages_enabled) guard_pages_enabled = thread->reguard_stack();