/*

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "jvm.h"

#include "classfile/systemDictionary.hpp"

#include "code/codeCache.hpp"

#include "code/debugInfoRec.hpp"

#include "code/nmethod.hpp"

#include "code/pcDesc.hpp"

#include "code/scopeDesc.hpp"

#include "interpreter/bytecode.hpp"

#include "interpreter/interpreter.hpp"

#include "interpreter/oopMapCache.hpp"

#include "memory/allocation.inline.hpp"

#include "memory/oopFactory.hpp"

#include "memory/resourceArea.hpp"

#include "oops/method.hpp"

#include "oops/objArrayOop.inline.hpp"

#include "oops/oop.inline.hpp"

#include "oops/fieldStreams.hpp"

#include "oops/typeArrayOop.inline.hpp"

#include "oops/verifyOopClosure.hpp"

#include "prims/jvmtiThreadState.hpp"

#include "runtime/biasedLocking.hpp"

#include "runtime/compilationPolicy.hpp"

#include "runtime/deoptimization.hpp"

#include "runtime/frame.inline.hpp"

#include "runtime/interfaceSupport.inline.hpp"

#include "runtime/safepointVerifiers.hpp"

#include "runtime/sharedRuntime.hpp"

#include "runtime/signature.hpp"

#include "runtime/stubRoutines.hpp"

#include "runtime/thread.hpp"

#include "runtime/threadSMR.hpp"

#include "runtime/vframe.hpp"

#include "runtime/vframeArray.hpp"

#include "runtime/vframe_hp.hpp"

#include "utilities/events.hpp"

#include "utilities/preserveException.hpp"

#include "utilities/xmlstream.hpp"

#if INCLUDE_JVMCI

#include "jvmci/jvmciRuntime.hpp"

#include "jvmci/jvmciJavaClasses.hpp"

#endif

bool DeoptimizationMarker::_is_active = false;

Deoptimization::UnrollBlock::UnrollBlock(int  size_of_deoptimized_frame,

                                         int  caller_adjustment,

                                         int  caller_actual_parameters,

                                         int  number_of_frames,

                                         intptr_t* frame_sizes,

                                         address* frame_pcs,

                                         BasicType return_type,

                                         int exec_mode) {

  _size_of_deoptimized_frame = size_of_deoptimized_frame;

  _caller_adjustment         = caller_adjustment;

  _caller_actual_parameters  = caller_actual_parameters;

  _number_of_frames          = number_of_frames;

  _frame_sizes               = frame_sizes;

  _frame_pcs                 = frame_pcs;

  _register_block            = NEW_C_HEAP_ARRAY(intptr_t, RegisterMap::reg_count * 2, mtCompiler);

  _return_type               = return_type;

  _initial_info              = 0;

  // PD (x86 only)

  _counter_temp              = 0;

  _unpack_kind               = exec_mode;

  _sender_sp_temp            = 0;

  _total_frame_sizes         = size_of_frames();

  assert(exec_mode >= 0 && exec_mode < Unpack_LIMIT, "Unexpected exec_mode");

}

Deoptimization::UnrollBlock::~UnrollBlock() {

  FREE_C_HEAP_ARRAY(intptr_t, _frame_sizes);

  FREE_C_HEAP_ARRAY(intptr_t, _frame_pcs);

  FREE_C_HEAP_ARRAY(intptr_t, _register_block);

}

intptr_t* Deoptimization::UnrollBlock::value_addr_at(int register_number) const {

  assert(register_number < RegisterMap::reg_count, "checking register number");

  return &_register_block[register_number * 2];

}

int Deoptimization::UnrollBlock::size_of_frames() const {

  // Acount first for the adjustment of the initial frame

  int result = _caller_adjustment;

  for (int index = 0; index < number_of_frames(); index++) {

    result += frame_sizes()[index];

  }

  return result;

}

void Deoptimization::UnrollBlock::print() {

  ttyLocker ttyl;

  tty->print_cr("UnrollBlock");

  tty->print_cr("  size_of_deoptimized_frame = %d", _size_of_deoptimized_frame);

  tty->print(   "  frame_sizes: ");

  for (int index = 0; index < number_of_frames(); index++) {

    tty->print(INTX_FORMAT " ", frame_sizes()[index]);

  }

  tty->cr();

}

// In order to make fetch_unroll_info work properly with escape

// analysis, The method was changed from JRT_LEAF to JRT_BLOCK_ENTRY and

// ResetNoHandleMark and HandleMark were removed from it. The actual reallocation

// of previously eliminated objects occurs in realloc_objects, which is

// called from the method fetch_unroll_info_helper below.

JRT_BLOCK_ENTRY(Deoptimization::UnrollBlock*, Deoptimization::fetch_unroll_info(JavaThread* thread, int exec_mode))

  // It is actually ok to allocate handles in a leaf method. It causes no safepoints,

  // but makes the entry a little slower. There is however a little dance we have to

  // do in debug mode to get around the NoHandleMark code in the JRT_LEAF macro

  // fetch_unroll_info() is called at the beginning of the deoptimization

  // handler. Note this fact before we start generating temporary frames

  // that can confuse an asynchronous stack walker. This counter is

  // decremented at the end of unpack_frames().

  if (TraceDeoptimization) {

    tty->print_cr("Deoptimizing thread " INTPTR_FORMAT, p2i(thread));

  }

  thread->inc_in_deopt_handler();

  return fetch_unroll_info_helper(thread, exec_mode);

JRT_END

// This is factored, since it is both called from a JRT_LEAF (deoptimization) and a JRT_ENTRY (uncommon_trap)

Deoptimization::UnrollBlock* Deoptimization::fetch_unroll_info_helper(JavaThread* thread, int exec_mode) {

  // Note: there is a safepoint safety issue here. No matter whether we enter

  // via vanilla deopt or uncommon trap we MUST NOT stop at a safepoint once

  // the vframeArray is created.

  //

  // Allocate our special deoptimization ResourceMark

  DeoptResourceMark* dmark = new DeoptResourceMark(thread);

  assert(thread->deopt_mark() == NULL, "Pending deopt!");

  thread->set_deopt_mark(dmark);

  frame stub_frame = thread->last_frame(); // Makes stack walkable as side effect

  RegisterMap map(thread, true);

  RegisterMap dummy_map(thread, false);

  // Now get the deoptee with a valid map

  frame deoptee = stub_frame.sender(&map);

  // Set the deoptee nmethod

  assert(thread->deopt_compiled_method() == NULL, "Pending deopt!");

  CompiledMethod* cm = deoptee.cb()->as_compiled_method_or_null();

  thread->set_deopt_compiled_method(cm);

  if (VerifyStack) {

    thread->validate_frame_layout();

  }

  // Create a growable array of VFrames where each VFrame represents an inlined

  // Java frame.  This storage is allocated with the usual system arena.

  assert(deoptee.is_compiled_frame(), "Wrong frame type");

  GrowableArray<compiledVFrame*>* chunk = new GrowableArray<compiledVFrame*>(10);

  vframe* vf = vframe::new_vframe(&deoptee, &map, thread);

  while (!vf->is_top()) {

    assert(vf->is_compiled_frame(), "Wrong frame type");

    chunk->push(compiledVFrame::cast(vf));

    vf = vf->sender();

  }

  assert(vf->is_compiled_frame(), "Wrong frame type");

  chunk->push(compiledVFrame::cast(vf));

  bool realloc_failures = false;

#if COMPILER2_OR_JVMCI

  // Reallocate the non-escaping objects and restore their fields. Then

  // relock objects if synchronization on them was eliminated.

#if !INCLUDE_JVMCI

  if (DoEscapeAnalysis || EliminateNestedLocks) {

    if (EliminateAllocations) {

#endif // INCLUDE_JVMCI

      assert (chunk->at(0)->scope() != NULL,"expect only compiled java frames");

      GrowableArray<ScopeValue*>* objects = chunk->at(0)->scope()->objects();

      // The flag return_oop() indicates call sites which return oop

      // in compiled code. Such sites include java method calls,

      // runtime calls (for example, used to allocate new objects/arrays

      // on slow code path) and any other calls generated in compiled code.

      // It is not guaranteed that we can get such information here only

      // by analyzing bytecode in deoptimized frames. This is why this flag

      // is set during method compilation (see Compile::Process_OopMap_Node()).

      // If the previous frame was popped or if we are dispatching an exception,

      // we don't have an oop result.

      bool save_oop_result = chunk->at(0)->scope()->return_oop() && !thread->popframe_forcing_deopt_reexecution() && (exec_mode == Unpack_deopt);

      Handle return_value;

      if (save_oop_result) {

        // Reallocation may trigger GC. If deoptimization happened on return from

        // call which returns oop we need to save it since it is not in oopmap.

        oop result = deoptee.saved_oop_result(&map);

        assert(oopDesc::is_oop_or_null(result), "must be oop");

        return_value = Handle(thread, result);

        assert(Universe::heap()->is_in_or_null(result), "must be heap pointer");

        if (TraceDeoptimization) {

          ttyLocker ttyl;

          tty->print_cr("SAVED OOP RESULT " INTPTR_FORMAT " in thread " INTPTR_FORMAT, p2i(result), p2i(thread));

        }

      }

      if (objects != NULL) {

        JRT_BLOCK

          realloc_failures = realloc_objects(thread, &deoptee, objects, THREAD);

        JRT_END

        bool skip_internal = (cm != NULL) && !cm->is_compiled_by_jvmci();

        reassign_fields(&deoptee, &map, objects, realloc_failures, skip_internal);

#ifndef PRODUCT

        if (TraceDeoptimization) {

          ttyLocker ttyl;

          tty->print_cr("REALLOC OBJECTS in thread " INTPTR_FORMAT, p2i(thread));

          print_objects(objects, realloc_failures);

        }

#endif

      }

      if (save_oop_result) {

        // Restore result.

        deoptee.set_saved_oop_result(&map, return_value());

      }

#if !INCLUDE_JVMCI

    }

    if (EliminateLocks) {

#endif // INCLUDE_JVMCI

#ifndef PRODUCT

      bool first = true;

#endif

      for (int i = 0; i < chunk->length(); i++) {

        compiledVFrame* cvf = chunk->at(i);

        assert (cvf->scope() != NULL,"expect only compiled java frames");

        GrowableArray<MonitorInfo*>* monitors = cvf->monitors();

        if (monitors->is_nonempty()) {

          relock_objects(monitors, thread, realloc_failures);

#ifndef PRODUCT

          if (PrintDeoptimizationDetails) {

            ttyLocker ttyl;

            for (int j = 0; j < monitors->length(); j++) {

              MonitorInfo* mi = monitors->at(j);

              if (mi->eliminated()) {

                if (first) {

                  first = false;

                  tty->print_cr("RELOCK OBJECTS in thread " INTPTR_FORMAT, p2i(thread));

                }

                if (mi->owner_is_scalar_replaced()) {

                  Klass* k = java_lang_Class::as_Klass(mi->owner_klass());

                  tty->print_cr("     failed reallocation for klass %s", k->external_name());

                } else {

                  tty->print_cr("     object <" INTPTR_FORMAT "> locked", p2i(mi->owner()));

                }

              }

            }

          }

#endif // !PRODUCT

        }

      }

#if !INCLUDE_JVMCI

    }

  }

#endif // INCLUDE_JVMCI

#endif // COMPILER2_OR_JVMCI

  ScopeDesc* trap_scope = chunk->at(0)->scope();

  Handle exceptionObject;

  if (trap_scope->rethrow_exception()) {

    if (PrintDeoptimizationDetails) {

      tty->print_cr("Exception to be rethrown in the interpreter for method %s::%s at bci %d", trap_scope->method()->method_holder()->name()->as_C_string(), trap_scope->method()->name()->as_C_string(), trap_scope->bci());

    }

    GrowableArray<ScopeValue*>* expressions = trap_scope->expressions();

    guarantee(expressions != NULL && expressions->length() > 0, "must have exception to throw");

    ScopeValue* topOfStack = expressions->top();

    exceptionObject = StackValue::create_stack_value(&deoptee, &map, topOfStack)->get_obj();

    guarantee(exceptionObject() != NULL, "exception oop can not be null");

  }

  // Ensure that no safepoint is taken after pointers have been stored

  // in fields of rematerialized objects.  If a safepoint occurs from here on

  // out the java state residing in the vframeArray will be missed.

  NoSafepointVerifier no_safepoint;

  vframeArray* array = create_vframeArray(thread, deoptee, &map, chunk, realloc_failures);

#if COMPILER2_OR_JVMCI

  if (realloc_failures) {

    pop_frames_failed_reallocs(thread, array);

  }

#endif

  assert(thread->vframe_array_head() == NULL, "Pending deopt!");

  thread->set_vframe_array_head(array);

  // Now that the vframeArray has been created if we have any deferred local writes

  // added by jvmti then we can free up that structure as the data is now in the

  // vframeArray

  if (thread->deferred_locals() != NULL) {

    GrowableArray<jvmtiDeferredLocalVariableSet*>* list = thread->deferred_locals();

    int i = 0;

    do {

      // Because of inlining we could have multiple vframes for a single frame

      // and several of the vframes could have deferred writes. Find them all.

      if (list->at(i)->id() == array->original().id()) {

        jvmtiDeferredLocalVariableSet* dlv = list->at(i);

        list->remove_at(i);

        // individual jvmtiDeferredLocalVariableSet are CHeapObj's

        delete dlv;

      } else {

        i++;

      }

    } while ( i < list->length() );

    if (list->length() == 0) {

      thread->set_deferred_locals(NULL);

      // free the list and elements back to C heap.

      delete list;

    }

  }

  // Compute the caller frame based on the sender sp of stub_frame and stored frame sizes info.

  CodeBlob* cb = stub_frame.cb();

  // Verify we have the right vframeArray

  assert(cb->frame_size() >= 0, "Unexpected frame size");

  intptr_t* unpack_sp = stub_frame.sp() + cb->frame_size();

  // If the deopt call site is a MethodHandle invoke call site we have

  // to adjust the unpack_sp.

  nmethod* deoptee_nm = deoptee.cb()->as_nmethod_or_null();

  if (deoptee_nm != NULL && deoptee_nm->is_method_handle_return(deoptee.pc()))

    unpack_sp = deoptee.unextended_sp();

#ifdef ASSERT

  assert(cb->is_deoptimization_stub() ||

         cb->is_uncommon_trap_stub() ||

         strcmp("Stub<DeoptimizationStub.deoptimizationHandler>", cb->name()) == 0 ||

         strcmp("Stub<UncommonTrapStub.uncommonTrapHandler>", cb->name()) == 0,

         "unexpected code blob: %s", cb->name());

#endif

  // This is a guarantee instead of an assert because if vframe doesn't match

  // we will unpack the wrong deoptimized frame and wind up in strange places

  // where it will be very difficult to figure out what went wrong. Better

  // to die an early death here than some very obscure death later when the

  // trail is cold.

  // Note: on ia64 this guarantee can be fooled by frames with no memory stack

  // in that it will fail to detect a problem when there is one. This needs

  // more work in tiger timeframe.

  guarantee(array->unextended_sp() == unpack_sp, "vframe_array_head must contain the vframeArray to unpack");

  int number_of_frames = array->frames();

  // Compute the vframes' sizes.  Note that frame_sizes[] entries are ordered from outermost to innermost

  // virtual activation, which is the reverse of the elements in the vframes array.

  intptr_t* frame_sizes = NEW_C_HEAP_ARRAY(intptr_t, number_of_frames, mtCompiler);

  // +1 because we always have an interpreter return address for the final slot.

  address* frame_pcs = NEW_C_HEAP_ARRAY(address, number_of_frames + 1, mtCompiler);

  int popframe_extra_args = 0;

  // Create an interpreter return address for the stub to use as its return

  // address so the skeletal frames are perfectly walkable

  frame_pcs[number_of_frames] = Interpreter::deopt_entry(vtos, 0);

  // PopFrame requires that the preserved incoming arguments from the recently-popped topmost

  // activation be put back on the expression stack of the caller for reexecution

  if (JvmtiExport::can_pop_frame() && thread->popframe_forcing_deopt_reexecution()) {

    popframe_extra_args = in_words(thread->popframe_preserved_args_size_in_words());

  }

  // Find the current pc for sender of the deoptee. Since the sender may have been deoptimized

  // itself since the deoptee vframeArray was created we must get a fresh value of the pc rather

  // than simply use array->sender.pc(). This requires us to walk the current set of frames

  //

  frame deopt_sender = stub_frame.sender(&dummy_map); // First is the deoptee frame

  deopt_sender = deopt_sender.sender(&dummy_map);     // Now deoptee caller

  // It's possible that the number of parameters at the call site is

  // different than number of arguments in the callee when method

  // handles are used.  If the caller is interpreted get the real

  // value so that the proper amount of space can be added to it's

  // frame.

  bool caller_was_method_handle = false;

  if (deopt_sender.is_interpreted_frame()) {

    methodHandle method = deopt_sender.interpreter_frame_method();

    Bytecode_invoke cur = Bytecode_invoke_check(method, deopt_sender.interpreter_frame_bci());

    if (cur.is_invokedynamic() || cur.is_invokehandle()) {

      // Method handle invokes may involve fairly arbitrary chains of

      // calls so it's impossible to know how much actual space the

      // caller has for locals.

      caller_was_method_handle = true;

    }

  }

  //

  // frame_sizes/frame_pcs[0] oldest frame (int or c2i)

  // frame_sizes/frame_pcs[1] next oldest frame (int)

  // frame_sizes/frame_pcs[n] youngest frame (int)

  //

  // Now a pc in frame_pcs is actually the return address to the frame's caller (a frame

  // owns the space for the return address to it's caller).  Confusing ain't it.

  //

  // The vframe array can address vframes with indices running from

  // 0.._frames-1. Index  0 is the youngest frame and _frame - 1 is the oldest (root) frame.

  // When we create the skeletal frames we need the oldest frame to be in the zero slot

  // in the frame_sizes/frame_pcs so the assembly code can do a trivial walk.

  // so things look a little strange in this loop.

  //

  int callee_parameters = 0;

  int callee_locals = 0;

  for (int index = 0; index < array->frames(); index++ ) {

    // frame[number_of_frames - 1 ] = on_stack_size(youngest)

    // frame[number_of_frames - 2 ] = on_stack_size(sender(youngest))

    // frame[number_of_frames - 3 ] = on_stack_size(sender(sender(youngest)))

    frame_sizes[number_of_frames - 1 - index] = BytesPerWord * array->element(index)->on_stack_size(callee_parameters,

                                                                                                    callee_locals,

                                                                                                    index == 0,

                                                                                                    popframe_extra_args);

    // This pc doesn't have to be perfect just good enough to identify the frame

    // as interpreted so the skeleton frame will be walkable

    // The correct pc will be set when the skeleton frame is completely filled out

    // The final pc we store in the loop is wrong and will be overwritten below

    frame_pcs[number_of_frames - 1 - index ] = Interpreter::deopt_entry(vtos, 0) - frame::pc_return_offset;

    callee_parameters = array->element(index)->method()->size_of_parameters();

    callee_locals = array->element(index)->method()->max_locals();

    popframe_extra_args = 0;

  }

  // Compute whether the root vframe returns a float or double value.

  BasicType return_type;

  {

    methodHandle method(thread, array->element(0)->method());

    Bytecode_invoke invoke = Bytecode_invoke_check(method, array->element(0)->bci());

    return_type = invoke.is_valid() ? invoke.result_type() : T_ILLEGAL;

  }

  // Compute information for handling adapters and adjusting the frame size of the caller.

  int caller_adjustment = 0;

  // Compute the amount the oldest interpreter frame will have to adjust

  // its caller's stack by. If the caller is a compiled frame then

  // we pretend that the callee has no parameters so that the

  // extension counts for the full amount of locals and not just

  // locals-parms. This is because without a c2i adapter the parm

  // area as created by the compiled frame will not be usable by

  // the interpreter. (Depending on the calling convention there

  // may not even be enough space).

  // QQQ I'd rather see this pushed down into last_frame_adjust

  // and have it take the sender (aka caller).

  if (deopt_sender.is_compiled_frame() || caller_was_method_handle) {

    caller_adjustment = last_frame_adjust(0, callee_locals);

  } else if (callee_locals > callee_parameters) {

    // The caller frame may need extending to accommodate

    // non-parameter locals of the first unpacked interpreted frame.

    // Compute that adjustment.

    caller_adjustment = last_frame_adjust(callee_parameters, callee_locals);

  }

  // If the sender is deoptimized the we must retrieve the address of the handler

  // since the frame will "magically" show the original pc before the deopt

  // and we'd undo the deopt.

  frame_pcs[0] = deopt_sender.raw_pc();

  assert(CodeCache::find_blob_unsafe(frame_pcs[0]) != NULL, "bad pc");

#if INCLUDE_JVMCI

  if (exceptionObject() != NULL) {

    thread->set_exception_oop(exceptionObject());

    exec_mode = Unpack_exception;

  }

#endif

  if (thread->frames_to_pop_failed_realloc() > 0 && exec_mode != Unpack_uncommon_trap) {

    assert(thread->has_pending_exception(), "should have thrown OOME");