1 /*

     2  * Copyright (c) 2003, 2014, Oracle and/or its affiliates. All rights reserved.

     3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

     4  *

     5  * This code is free software; you can redistribute it and/or modify it

     6  * under the terms of the GNU General Public License version 2 only, as

     7  * published by the Free Software Foundation.

     8  *

     9  * This code is distributed in the hope that it will be useful, but WITHOUT

    10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    12  * version 2 for more details (a copy is included in the LICENSE file that

    13  * accompanied this code).

    14  *

    15  * You should have received a copy of the GNU General Public License version

    16  * 2 along with this work; if not, write to the Free Software Foundation,

    17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    18  *

    19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

    20  * or visit www.oracle.com if you need additional information or have any

    21  * questions.

    22  *

    23  */

    24 

    25 #include "precompiled.hpp"

    26 #include "interp_masm_x86.hpp"

    27 #include "interpreter/interpreter.hpp"

    28 #include "interpreter/interpreterRuntime.hpp"

    29 #include "oops/arrayOop.hpp"

    30 #include "oops/markOop.hpp"

    31 #include "oops/methodData.hpp"

    32 #include "oops/method.hpp"

    33 #include "prims/jvmtiExport.hpp"

    34 #include "prims/jvmtiRedefineClassesTrace.hpp"

    35 #include "prims/jvmtiThreadState.hpp"

    36 #include "runtime/basicLock.hpp"

    37 #include "runtime/biasedLocking.hpp"

    38 #include "runtime/sharedRuntime.hpp"

    39 #include "runtime/thread.inline.hpp"

    40 

    41 

    42 // Implementation of InterpreterMacroAssembler

    43 

    44 #ifdef CC_INTERP

    45 void InterpreterMacroAssembler::get_method(Register reg) {

    46   movptr(reg, Address(rbp, -((int)sizeof(BytecodeInterpreter) + 2 * wordSize)));

    47   movptr(reg, Address(reg, byte_offset_of(BytecodeInterpreter, _method)));

    48 }

    49 #endif // CC_INTERP

    50 

    51 #ifndef CC_INTERP

    52 

    53 void InterpreterMacroAssembler::call_VM_leaf_base(address entry_point,

    54                                                   int number_of_arguments) {

    55   // interpreter specific

    56   //

    57   // Note: No need to save/restore bcp & locals (r13 & r14) pointer

    58   //       since these are callee saved registers and no blocking/

    59   //       GC can happen in leaf calls.

    60   // Further Note: DO NOT save/restore bcp/locals. If a caller has

    61   // already saved them so that it can use esi/edi as temporaries

    62   // then a save/restore here will DESTROY the copy the caller

    63   // saved! There used to be a save_bcp() that only happened in

    64   // the ASSERT path (no restore_bcp). Which caused bizarre failures

    65   // when jvm built with ASSERTs.

    66 #ifdef ASSERT

    67   {

    68     Label L;

    69     cmpptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);

    70     jcc(Assembler::equal, L);

    71     stop("InterpreterMacroAssembler::call_VM_leaf_base:"

    72          " last_sp != NULL");

    73     bind(L);

    74   }

    75 #endif

    76   // super call

    77   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);

    78   // interpreter specific

    79   // Used to ASSERT that r13/r14 were equal to frame's bcp/locals

    80   // but since they may not have been saved (and we don't want to

    81   // save thme here (see note above) the assert is invalid.

    82 }

    83 

    84 void InterpreterMacroAssembler::call_VM_base(Register oop_result,

    85                                              Register java_thread,

    86                                              Register last_java_sp,

    87                                              address  entry_point,

    88                                              int      number_of_arguments,

    89                                              bool     check_exceptions) {

    90   // interpreter specific

    91   //

    92   // Note: Could avoid restoring locals ptr (callee saved) - however doesn't

    93   //       really make a difference for these runtime calls, since they are

    94   //       slow anyway. Btw., bcp must be saved/restored since it may change

    95   //       due to GC.

    96   // assert(java_thread == noreg , "not expecting a precomputed java thread");

    97   save_bcp();

    98 #ifdef ASSERT

    99   {

   100     Label L;

   101     cmpptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);

   102     jcc(Assembler::equal, L);

   103     stop("InterpreterMacroAssembler::call_VM_leaf_base:"

   104          " last_sp != NULL");

   105     bind(L);

   106   }

   107 #endif /* ASSERT */

   108   // super call

   109   MacroAssembler::call_VM_base(oop_result, noreg, last_java_sp,

   110                                entry_point, number_of_arguments,

   111                                check_exceptions);

   112   // interpreter specific

   113   restore_bcp();

   114   restore_locals();

   115 }

   116 

   117 

   118 void InterpreterMacroAssembler::check_and_handle_popframe(Register java_thread) {

   119   if (JvmtiExport::can_pop_frame()) {

   120     Label L;

   121     // Initiate popframe handling only if it is not already being

   122     // processed.  If the flag has the popframe_processing bit set, it

   123     // means that this code is called *during* popframe handling - we

   124     // don't want to reenter.

   125     // This method is only called just after the call into the vm in

   126     // call_VM_base, so the arg registers are available.

   127     movl(c_rarg0, Address(r15_thread, JavaThread::popframe_condition_offset()));

   128     testl(c_rarg0, JavaThread::popframe_pending_bit);

   129     jcc(Assembler::zero, L);

   130     testl(c_rarg0, JavaThread::popframe_processing_bit);

   131     jcc(Assembler::notZero, L);

   132     // Call Interpreter::remove_activation_preserving_args_entry() to get the

   133     // address of the same-named entrypoint in the generated interpreter code.

   134     call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_preserving_args_entry));

   135     jmp(rax);

   136     bind(L);

   137   }

   138 }

   139 

   140 

   141 void InterpreterMacroAssembler::load_earlyret_value(TosState state) {

   142   movptr(rcx, Address(r15_thread, JavaThread::jvmti_thread_state_offset()));

   143   const Address tos_addr(rcx, JvmtiThreadState::earlyret_tos_offset());

   144   const Address oop_addr(rcx, JvmtiThreadState::earlyret_oop_offset());

   145   const Address val_addr(rcx, JvmtiThreadState::earlyret_value_offset());

   146   switch (state) {

   147     case atos: movptr(rax, oop_addr);

   148                movptr(oop_addr, (int32_t)NULL_WORD);

   149                verify_oop(rax, state);              break;

   150     case ltos: movptr(rax, val_addr);                 break;

   151     case btos:                                   // fall through

   152     case ctos:                                   // fall through

   153     case stos:                                   // fall through

   154     case itos: movl(rax, val_addr);                 break;

   155     case ftos: movflt(xmm0, val_addr);              break;

   156     case dtos: movdbl(xmm0, val_addr);              break;

   157     case vtos: /* nothing to do */                  break;

   158     default  : ShouldNotReachHere();

   159   }

   160   // Clean up tos value in the thread object

   161   movl(tos_addr,  (int) ilgl);

   162   movl(val_addr,  (int32_t) NULL_WORD);

   163 }

   164 

   165 

   166 void InterpreterMacroAssembler::check_and_handle_earlyret(Register java_thread) {

   167   if (JvmtiExport::can_force_early_return()) {

   168     Label L;

   169     movptr(c_rarg0, Address(r15_thread, JavaThread::jvmti_thread_state_offset()));

   170     testptr(c_rarg0, c_rarg0);

   171     jcc(Assembler::zero, L); // if (thread->jvmti_thread_state() == NULL) exit;

   172 

   173     // Initiate earlyret handling only if it is not already being processed.

   174     // If the flag has the earlyret_processing bit set, it means that this code

   175     // is called *during* earlyret handling - we don't want to reenter.

   176     movl(c_rarg0, Address(c_rarg0, JvmtiThreadState::earlyret_state_offset()));

   177     cmpl(c_rarg0, JvmtiThreadState::earlyret_pending);

   178     jcc(Assembler::notEqual, L);

   179 

   180     // Call Interpreter::remove_activation_early_entry() to get the address of the

   181     // same-named entrypoint in the generated interpreter code.

   182     movptr(c_rarg0, Address(r15_thread, JavaThread::jvmti_thread_state_offset()));

   183     movl(c_rarg0, Address(c_rarg0, JvmtiThreadState::earlyret_tos_offset()));

   184     call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_early_entry), c_rarg0);

   185     jmp(rax);

   186     bind(L);

   187   }

   188 }

   189 

   190 

   191 void InterpreterMacroAssembler::get_unsigned_2_byte_index_at_bcp(

   192   Register reg,

   193   int bcp_offset) {

   194   assert(bcp_offset >= 0, "bcp is still pointing to start of bytecode");

   195   load_unsigned_short(reg, Address(r13, bcp_offset));

   196   bswapl(reg);

   197   shrl(reg, 16);

   198 }

   199 

   200 

   201 void InterpreterMacroAssembler::get_cache_index_at_bcp(Register index,

   202                                                        int bcp_offset,

   203                                                        size_t index_size) {

   204   assert(bcp_offset > 0, "bcp is still pointing to start of bytecode");

   205   if (index_size == sizeof(u2)) {

   206     load_unsigned_short(index, Address(r13, bcp_offset));

   207   } else if (index_size == sizeof(u4)) {

   208     movl(index, Address(r13, bcp_offset));

   209     // Check if the secondary index definition is still ~x, otherwise

   210     // we have to change the following assembler code to calculate the

   211     // plain index.

   212     assert(ConstantPool::decode_invokedynamic_index(~123) == 123, "else change next line");

   213     notl(index);  // convert to plain index

   214   } else if (index_size == sizeof(u1)) {

   215     load_unsigned_byte(index, Address(r13, bcp_offset));

   216   } else {

   217     ShouldNotReachHere();

   218   }

   219 }

   220 

   221 

   222 void InterpreterMacroAssembler::get_cache_and_index_at_bcp(Register cache,

   223                                                            Register index,

   224                                                            int bcp_offset,

   225                                                            size_t index_size) {

   226   assert_different_registers(cache, index);

   227   get_cache_index_at_bcp(index, bcp_offset, index_size);

   228   movptr(cache, Address(rbp, frame::interpreter_frame_cache_offset * wordSize));

   229   assert(sizeof(ConstantPoolCacheEntry) == 4 * wordSize, "adjust code below");

   230   // convert from field index to ConstantPoolCacheEntry index

   231   assert(exact_log2(in_words(ConstantPoolCacheEntry::size())) == 2, "else change next line");

   232   shll(index, 2);

   233 }

   234 

   235 

   236 void InterpreterMacroAssembler::get_cache_and_index_and_bytecode_at_bcp(Register cache,

   237                                                                         Register index,

   238                                                                         Register bytecode,

   239                                                                         int byte_no,

   240                                                                         int bcp_offset,

   241                                                                         size_t index_size) {

   242   get_cache_and_index_at_bcp(cache, index, bcp_offset, index_size);

   243   // We use a 32-bit load here since the layout of 64-bit words on

   244   // little-endian machines allow us that.

   245   movl(bytecode, Address(cache, index, Address::times_ptr, ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::indices_offset()));

   246   const int shift_count = (1 + byte_no) * BitsPerByte;

   247   assert((byte_no == TemplateTable::f1_byte && shift_count == ConstantPoolCacheEntry::bytecode_1_shift) ||

   248          (byte_no == TemplateTable::f2_byte && shift_count == ConstantPoolCacheEntry::bytecode_2_shift),

   249          "correct shift count");

   250   shrl(bytecode, shift_count);

   251   assert(ConstantPoolCacheEntry::bytecode_1_mask == ConstantPoolCacheEntry::bytecode_2_mask, "common mask");

   252   andl(bytecode, ConstantPoolCacheEntry::bytecode_1_mask);

   253 }

   254 

   255 

   256 void InterpreterMacroAssembler::get_cache_entry_pointer_at_bcp(Register cache,

   257                                                                Register tmp,

   258                                                                int bcp_offset,

   259                                                                size_t index_size) {

   260   assert(cache != tmp, "must use different register");

   261   get_cache_index_at_bcp(tmp, bcp_offset, index_size);

   262   assert(sizeof(ConstantPoolCacheEntry) == 4 * wordSize, "adjust code below");

   263   // convert from field index to ConstantPoolCacheEntry index

   264   // and from word offset to byte offset

   265   assert(exact_log2(in_bytes(ConstantPoolCacheEntry::size_in_bytes())) == 2 + LogBytesPerWord, "else change next line");

   266   shll(tmp, 2 + LogBytesPerWord);

   267   movptr(cache, Address(rbp, frame::interpreter_frame_cache_offset * wordSize));

   268   // skip past the header

   269   addptr(cache, in_bytes(ConstantPoolCache::base_offset()));

   270   addptr(cache, tmp);  // construct pointer to cache entry

   271 }

   272 

   273 // Load object from cpool->resolved_references(index)

   274 void InterpreterMacroAssembler::load_resolved_reference_at_index(

   275                                            Register result, Register index) {

   276   assert_different_registers(result, index);

   277   // convert from field index to resolved_references() index and from

   278   // word index to byte offset. Since this is a java object, it can be compressed

   279   Register tmp = index;  // reuse

   280   shll(tmp, LogBytesPerHeapOop);

   281 

   282   get_constant_pool(result);

   283   // load pointer for resolved_references[] objArray

   284   movptr(result, Address(result, ConstantPool::resolved_references_offset_in_bytes()));

   285   // JNIHandles::resolve(obj);

   286   movptr(result, Address(result, 0));

   287   // Add in the index

   288   addptr(result, tmp);

   289   load_heap_oop(result, Address(result, arrayOopDesc::base_offset_in_bytes(T_OBJECT)));

   290 }

   291 

   292 // Generate a subtype check: branch to ok_is_subtype if sub_klass is a

   293 // subtype of super_klass.

   294 //

   295 // Args:

   296 //      rax: superklass

   297 //      Rsub_klass: subklass

   298 //

   299 // Kills:

   300 //      rcx, rdi

   301 void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,

   302                                                   Label& ok_is_subtype) {

   303   assert(Rsub_klass != rax, "rax holds superklass");

   304   assert(Rsub_klass != r14, "r14 holds locals");

   305   assert(Rsub_klass != r13, "r13 holds bcp");

   306   assert(Rsub_klass != rcx, "rcx holds 2ndary super array length");

   307   assert(Rsub_klass != rdi, "rdi holds 2ndary super array scan ptr");

   308 

   309   // Profile the not-null value's klass.

   310   profile_typecheck(rcx, Rsub_klass, rdi); // blows rcx, reloads rdi

   311 

   312   // Do the check.

   313   check_klass_subtype(Rsub_klass, rax, rcx, ok_is_subtype); // blows rcx

   314 

   315   // Profile the failure of the check.

   316   profile_typecheck_failed(rcx); // blows rcx

   317 }

   318 

   319 

   320 

   321 // Java Expression Stack

   322 

   323 void InterpreterMacroAssembler::pop_ptr(Register r) {

   324   pop(r);

   325 }

   326 

   327 void InterpreterMacroAssembler::pop_i(Register r) {

   328   // XXX can't use pop currently, upper half non clean

   329   movl(r, Address(rsp, 0));

   330   addptr(rsp, wordSize);

   331 }

   332 

   333 void InterpreterMacroAssembler::pop_l(Register r) {

   334   movq(r, Address(rsp, 0));

   335   addptr(rsp, 2 * Interpreter::stackElementSize);

   336 }

   337 

   338 void InterpreterMacroAssembler::pop_f(XMMRegister r) {

   339   movflt(r, Address(rsp, 0));

   340   addptr(rsp, wordSize);

   341 }

   342 

   343 void InterpreterMacroAssembler::pop_d(XMMRegister r) {

   344   movdbl(r, Address(rsp, 0));

   345   addptr(rsp, 2 * Interpreter::stackElementSize);

   346 }

   347 

   348 void InterpreterMacroAssembler::push_ptr(Register r) {

   349   push(r);

   350 }

   351 

   352 void InterpreterMacroAssembler::push_i(Register r) {

   353   push(r);

   354 }

   355 

   356 void InterpreterMacroAssembler::push_l(Register r) {

   357   subptr(rsp, 2 * wordSize);

   358   movq(Address(rsp, 0), r);

   359 }

   360 

   361 void InterpreterMacroAssembler::push_f(XMMRegister r) {

   362   subptr(rsp, wordSize);

   363   movflt(Address(rsp, 0), r);

   364 }

   365 

   366 void InterpreterMacroAssembler::push_d(XMMRegister r) {

   367   subptr(rsp, 2 * wordSize);

   368   movdbl(Address(rsp, 0), r);

   369 }

   370 

   371 void InterpreterMacroAssembler::pop(TosState state) {

   372   switch (state) {

   373   case atos: pop_ptr();                 break;

   374   case btos:

   375   case ctos:

   376   case stos:

   377   case itos: pop_i();                   break;

   378   case ltos: pop_l();                   break;

   379   case ftos: pop_f();                   break;

   380   case dtos: pop_d();                   break;

   381   case vtos: /* nothing to do */        break;

   382   default:   ShouldNotReachHere();

   383   }

   384   verify_oop(rax, state);

   385 }

   386 

   387 void InterpreterMacroAssembler::push(TosState state) {

   388   verify_oop(rax, state);

   389   switch (state) {

   390   case atos: push_ptr();                break;

   391   case btos:

   392   case ctos:

   393   case stos:

   394   case itos: push_i();                  break;

   395   case ltos: push_l();                  break;

   396   case ftos: push_f();                  break;

   397   case dtos: push_d();                  break;

   398   case vtos: /* nothing to do */        break;

   399   default  : ShouldNotReachHere();

   400   }

   401 }

   402 

   403 

   404 // Helpers for swap and dup

   405 void InterpreterMacroAssembler::load_ptr(int n, Register val) {

   406   movptr(val, Address(rsp, Interpreter::expr_offset_in_bytes(n)));

   407 }

   408 

   409 void InterpreterMacroAssembler::store_ptr(int n, Register val) {

   410   movptr(Address(rsp, Interpreter::expr_offset_in_bytes(n)), val);

   411 }

   412 

   413 

   414 void InterpreterMacroAssembler::prepare_to_jump_from_interpreted() {

   415   // set sender sp

   416   lea(r13, Address(rsp, wordSize));

   417   // record last_sp

   418   movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), r13);

   419 }

   420 

   421 

   422 // Jump to from_interpreted entry of a call unless single stepping is possible

   423 // in this thread in which case we must call the i2i entry

   424 void InterpreterMacroAssembler::jump_from_interpreted(Register method, Register temp) {

   425   prepare_to_jump_from_interpreted();

   426 

   427   if (JvmtiExport::can_post_interpreter_events()) {

   428     Label run_compiled_code;

   429     // JVMTI events, such as single-stepping, are implemented partly by avoiding running

   430     // compiled code in threads for which the event is enabled.  Check here for

   431     // interp_only_mode if these events CAN be enabled.

   432     // interp_only is an int, on little endian it is sufficient to test the byte only

   433     // Is a cmpl faster?

   434     cmpb(Address(r15_thread, JavaThread::interp_only_mode_offset()), 0);

   435     jccb(Assembler::zero, run_compiled_code);

   436     jmp(Address(method, Method::interpreter_entry_offset()));

   437     bind(run_compiled_code);

   438   }

   439 

   440   jmp(Address(method, Method::from_interpreted_offset()));

   441 

   442 }

   443 

   444 

   445 // The following two routines provide a hook so that an implementation

   446 // can schedule the dispatch in two parts.  amd64 does not do this.

   447 void InterpreterMacroAssembler::dispatch_prolog(TosState state, int step) {

   448   // Nothing amd64 specific to be done here

   449 }

   450 

   451 void InterpreterMacroAssembler::dispatch_epilog(TosState state, int step) {

   452   dispatch_next(state, step);

   453 }

   454 

   455 void InterpreterMacroAssembler::dispatch_base(TosState state,

   456                                               address* table,

   457                                               bool verifyoop) {

   458   verify_FPU(1, state);

   459   if (VerifyActivationFrameSize) {

   460     Label L;

   461     mov(rcx, rbp);

   462     subptr(rcx, rsp);

   463     int32_t min_frame_size =

   464       (frame::link_offset - frame::interpreter_frame_initial_sp_offset) *

   465       wordSize;

   466     cmpptr(rcx, (int32_t)min_frame_size);

   467     jcc(Assembler::greaterEqual, L);

   468     stop("broken stack frame");

   469     bind(L);

   470   }

   471   if (verifyoop) {

   472     verify_oop(rax, state);

   473   }

   474   lea(rscratch1, ExternalAddress((address)table));

   475   jmp(Address(rscratch1, rbx, Address::times_8));

   476 }

   477 

   478 void InterpreterMacroAssembler::dispatch_only(TosState state) {

   479   dispatch_base(state, Interpreter::dispatch_table(state));

   480 }

   481 

   482 void InterpreterMacroAssembler::dispatch_only_normal(TosState state) {

   483   dispatch_base(state, Interpreter::normal_table(state));

   484 }

   485 

   486 void InterpreterMacroAssembler::dispatch_only_noverify(TosState state) {

   487   dispatch_base(state, Interpreter::normal_table(state), false);

   488 }

   489 

   490 

   491 void InterpreterMacroAssembler::dispatch_next(TosState state, int step) {

   492   // load next bytecode (load before advancing r13 to prevent AGI)

   493   load_unsigned_byte(rbx, Address(r13, step));

   494   // advance r13

   495   increment(r13, step);

   496   dispatch_base(state, Interpreter::dispatch_table(state));

   497 }

   498 

   499 void InterpreterMacroAssembler::dispatch_via(TosState state, address* table) {

   500   // load current bytecode

   501   load_unsigned_byte(rbx, Address(r13, 0));

   502   dispatch_base(state, table);

   503 }

   504 

   505 // remove activation

   506 //

   507 // Unlock the receiver if this is a synchronized method.

   508 // Unlock any Java monitors from syncronized blocks.

   509 // Remove the activation from the stack.

   510 //

   511 // If there are locked Java monitors

   512 //    If throw_monitor_exception

   513 //       throws IllegalMonitorStateException

   514 //    Else if install_monitor_exception

   515 //       installs IllegalMonitorStateException

   516 //    Else

   517 //       no error processing

   518 void InterpreterMacroAssembler::remove_activation(

   519         TosState state,

   520         Register ret_addr,

   521         bool throw_monitor_exception,

   522         bool install_monitor_exception,

   523         bool notify_jvmdi) {

   524   // Note: Registers rdx xmm0 may be in use for the

   525   // result check if synchronized method

   526   Label unlocked, unlock, no_unlock;

   527 

   528   // get the value of _do_not_unlock_if_synchronized into rdx

   529   const Address do_not_unlock_if_synchronized(r15_thread,

   530     in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));

   531   movbool(rdx, do_not_unlock_if_synchronized);

   532   movbool(do_not_unlock_if_synchronized, false); // reset the flag

   533 

   534  // get method access flags

   535   movptr(rbx, Address(rbp, frame::interpreter_frame_method_offset * wordSize));

   536   movl(rcx, Address(rbx, Method::access_flags_offset()));

   537   testl(rcx, JVM_ACC_SYNCHRONIZED);

   538   jcc(Assembler::zero, unlocked);

   539 

   540   // Don't unlock anything if the _do_not_unlock_if_synchronized flag

   541   // is set.

   542   testbool(rdx);

   543   jcc(Assembler::notZero, no_unlock);

   544 

   545   // unlock monitor

   546   push(state); // save result

   547 

   548   // BasicObjectLock will be first in list, since this is a

   549   // synchronized method. However, need to check that the object has

   550   // not been unlocked by an explicit monitorexit bytecode.

   551   const Address monitor(rbp, frame::interpreter_frame_initial_sp_offset *

   552                         wordSize - (int) sizeof(BasicObjectLock));

   553   // We use c_rarg1 so that if we go slow path it will be the correct

   554   // register for unlock_object to pass to VM directly

   555   lea(c_rarg1, monitor); // address of first monitor

   556 

   557   movptr(rax, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));

   558   testptr(rax, rax);

   559   jcc(Assembler::notZero, unlock);

   560 

   561   pop(state);

   562   if (throw_monitor_exception) {

   563     // Entry already unlocked, need to throw exception

   564     call_VM(noreg, CAST_FROM_FN_PTR(address,

   565                    InterpreterRuntime::throw_illegal_monitor_state_exception));

   566     should_not_reach_here();

   567   } else {

   568     // Monitor already unlocked during a stack unroll. If requested,

   569     // install an illegal_monitor_state_exception.  Continue with

   570     // stack unrolling.

   571     if (install_monitor_exception) {

   572       call_VM(noreg, CAST_FROM_FN_PTR(address,

   573                      InterpreterRuntime::new_illegal_monitor_state_exception));

   574     }

   575     jmp(unlocked);

   576   }

   577 

   578   bind(unlock);

   579   unlock_object(c_rarg1);

   580   pop(state);

   581 

   582   // Check that for block-structured locking (i.e., that all locked

   583   // objects has been unlocked)

   584   bind(unlocked);

   585 

   586   // rax: Might contain return value

   587 

   588   // Check that all monitors are unlocked

   589   {

   590     Label loop, exception, entry, restart;

   591     const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;

   592     const Address monitor_block_top(

   593         rbp, frame::interpreter_frame_monitor_block_top_offset * wordSize);

   594     const Address monitor_block_bot(

   595         rbp, frame::interpreter_frame_initial_sp_offset * wordSize);

   596 

   597     bind(restart);

   598     // We use c_rarg1 so that if we go slow path it will be the correct

   599     // register for unlock_object to pass to VM directly

   600     movptr(c_rarg1, monitor_block_top); // points to current entry, starting

   601                                   // with top-most entry

   602     lea(rbx, monitor_block_bot);  // points to word before bottom of

   603                                   // monitor block

   604     jmp(entry);

   605 

   606     // Entry already locked, need to throw exception

   607     bind(exception);

   608 

   609     if (throw_monitor_exception) {

   610       // Throw exception

   611       MacroAssembler::call_VM(noreg,

   612                               CAST_FROM_FN_PTR(address, InterpreterRuntime::

   613                                    throw_illegal_monitor_state_exception));

   614       should_not_reach_here();

   615     } else {

   616       // Stack unrolling. Unlock object and install illegal_monitor_exception.

   617       // Unlock does not block, so don't have to worry about the frame.

   618       // We don't have to preserve c_rarg1 since we are going to throw an exception.

   619 

   620       push(state);

   621       unlock_object(c_rarg1);

   622       pop(state);

   623 

   624       if (install_monitor_exception) {

   625         call_VM(noreg, CAST_FROM_FN_PTR(address,

   626                                         InterpreterRuntime::

   627                                         new_illegal_monitor_state_exception));

   628       }

   629 

   630       jmp(restart);

   631     }

   632 

   633     bind(loop);

   634     // check if current entry is used

   635     cmpptr(Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()), (int32_t) NULL);

   636     jcc(Assembler::notEqual, exception);

   637 

   638     addptr(c_rarg1, entry_size); // otherwise advance to next entry

   639     bind(entry);

   640     cmpptr(c_rarg1, rbx); // check if bottom reached

   641     jcc(Assembler::notEqual, loop); // if not at bottom then check this entry

   642   }

   643 

   644   bind(no_unlock);

   645 

   646   // jvmti support

   647   if (notify_jvmdi) {

   648     notify_method_exit(state, NotifyJVMTI);    // preserve TOSCA

   649   } else {

   650     notify_method_exit(state, SkipNotifyJVMTI); // preserve TOSCA

   651   }

   652 

   653   // remove activation

   654   // get sender sp

   655   movptr(rbx,

   656          Address(rbp, frame::interpreter_frame_sender_sp_offset * wordSize));

   657   leave();                           // remove frame anchor

   658   pop(ret_addr);                     // get return address

   659   mov(rsp, rbx);                     // set sp to sender sp

   660 }

   661 

   662 #endif // C_INTERP

   663 

   664 void InterpreterMacroAssembler::get_method_counters(Register method,

   665                                                     Register mcs, Label& skip) {

   666   Label has_counters;

   667   movptr(mcs, Address(method, Method::method_counters_offset()));

   668   testptr(mcs, mcs);

   669   jcc(Assembler::notZero, has_counters);

   670   call_VM(noreg, CAST_FROM_FN_PTR(address,

   671           InterpreterRuntime::build_method_counters), method);

   672   movptr(mcs, Address(method,Method::method_counters_offset()));

   673   testptr(mcs, mcs);

   674   jcc(Assembler::zero, skip); // No MethodCounters allocated, OutOfMemory

   675   bind(has_counters);

   676 }

   677 

   678 

   679 // Lock object

   680 //

   681 // Args:

   682 //      c_rarg1: BasicObjectLock to be used for locking

   683 //

   684 // Kills:

   685 //      rax

   686 //      c_rarg0, c_rarg1, c_rarg2, c_rarg3, .. (param regs)

   687 //      rscratch1, rscratch2 (scratch regs)

   688 void InterpreterMacroAssembler::lock_object(Register lock_reg) {

   689   assert(lock_reg == c_rarg1, "The argument is only for looks. It must be c_rarg1");

   690 

   691   if (UseHeavyMonitors) {

   692     call_VM(noreg,

   693             CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),

   694             lock_reg);

   695   } else {

   696     Label done;

   697 

   698     const Register swap_reg = rax; // Must use rax for cmpxchg instruction

   699     const Register obj_reg = c_rarg3; // Will contain the oop

   700 

   701     const int obj_offset = BasicObjectLock::obj_offset_in_bytes();

   702     const int lock_offset = BasicObjectLock::lock_offset_in_bytes ();

   703     const int mark_offset = lock_offset +

   704                             BasicLock::displaced_header_offset_in_bytes();

   705 

   706     Label slow_case;

   707 

   708     // Load object pointer into obj_reg %c_rarg3

   709     movptr(obj_reg, Address(lock_reg, obj_offset));

   710 

   711     if (UseBiasedLocking) {

   712       biased_locking_enter(lock_reg, obj_reg, swap_reg, rscratch1, false, done, &slow_case);

   713     }

   714 

   715     // Load immediate 1 into swap_reg %rax

   716     movl(swap_reg, 1);

   717 

   718     // Load (object->mark() | 1) into swap_reg %rax

   719     orptr(swap_reg, Address(obj_reg, 0));

   720 

   721     // Save (object->mark() | 1) into BasicLock's displaced header

   722     movptr(Address(lock_reg, mark_offset), swap_reg);

   723 

   724     assert(lock_offset == 0,

   725            "displached header must be first word in BasicObjectLock");

   726 

   727     if (os::is_MP()) lock();

   728     cmpxchgptr(lock_reg, Address(obj_reg, 0));

   729     if (PrintBiasedLockingStatistics) {

   730       cond_inc32(Assembler::zero,

   731                  ExternalAddress((address) BiasedLocking::fast_path_entry_count_addr()));

   732     }

   733     jcc(Assembler::zero, done);

   734 

   735     // Test if the oopMark is an obvious stack pointer, i.e.,

   736     //  1) (mark & 7) == 0, and

   737     //  2) rsp <= mark < mark + os::pagesize()

   738     //

   739     // These 3 tests can be done by evaluating the following

   740     // expression: ((mark - rsp) & (7 - os::vm_page_size())),

   741     // assuming both stack pointer and pagesize have their

   742     // least significant 3 bits clear.

   743     // NOTE: the oopMark is in swap_reg %rax as the result of cmpxchg

   744     subptr(swap_reg, rsp);

   745     andptr(swap_reg, 7 - os::vm_page_size());

   746 

   747     // Save the test result, for recursive case, the result is zero

   748     movptr(Address(lock_reg, mark_offset), swap_reg);

   749 

   750     if (PrintBiasedLockingStatistics) {

   751       cond_inc32(Assembler::zero,

   752                  ExternalAddress((address) BiasedLocking::fast_path_entry_count_addr()));

   753     }

   754     jcc(Assembler::zero, done);

   755 

   756     bind(slow_case);

   757 

   758     // Call the runtime routine for slow case

   759     call_VM(noreg,

   760             CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),

   761             lock_reg);

   762 

   763     bind(done);

   764   }

   765 }

   766 

   767 

   768 // Unlocks an object. Used in monitorexit bytecode and

   769 // remove_activation.  Throws an IllegalMonitorException if object is

   770 // not locked by current thread.

   771 //

   772 // Args:

   773 //      c_rarg1: BasicObjectLock for lock

   774 //

   775 // Kills:

   776 //      rax

   777 //      c_rarg0, c_rarg1, c_rarg2, c_rarg3, ... (param regs)

   778 //      rscratch1, rscratch2 (scratch regs)

   779 void InterpreterMacroAssembler::unlock_object(Register lock_reg) {

   780   assert(lock_reg == c_rarg1, "The argument is only for looks. It must be rarg1");

   781 

   782   if (UseHeavyMonitors) {

   783     call_VM(noreg,

   784             CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit),

   785             lock_reg);

   786   } else {

   787     Label done;

   788 

   789     const Register swap_reg   = rax;  // Must use rax for cmpxchg instruction

   790     const Register header_reg = c_rarg2;  // Will contain the old oopMark

   791     const Register obj_reg    = c_rarg3;  // Will contain the oop

   792 

   793     save_bcp(); // Save in case of exception

   794 

   795     // Convert from BasicObjectLock structure to object and BasicLock

   796     // structure Store the BasicLock address into %rax

   797     lea(swap_reg, Address(lock_reg, BasicObjectLock::lock_offset_in_bytes()));

   798 

   799     // Load oop into obj_reg(%c_rarg3)

   800     movptr(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));

   801 

   802     // Free entry

   803     movptr(Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()), (int32_t)NULL_WORD);

   804 

   805     if (UseBiasedLocking) {

   806       biased_locking_exit(obj_reg, header_reg, done);

   807     }

   808 

   809     // Load the old header from BasicLock structure

   810     movptr(header_reg, Address(swap_reg,

   811                                BasicLock::displaced_header_offset_in_bytes()));

   812 

   813     // Test for recursion

   814     testptr(header_reg, header_reg);

   815 

   816     // zero for recursive case

   817     jcc(Assembler::zero, done);

   818 

   819     // Atomic swap back the old header

   820     if (os::is_MP()) lock();

   821     cmpxchgptr(header_reg, Address(obj_reg, 0));

   822 

   823     // zero for recursive case

   824     jcc(Assembler::zero, done);

   825 

   826     // Call the runtime routine for slow case.

   827     movptr(Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()),

   828          obj_reg); // restore obj

   829     call_VM(noreg,

   830             CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit),

   831             lock_reg);

   832 

   833     bind(done);

   834 

   835     restore_bcp();

   836   }

   837 }

   838 

   839 #ifndef CC_INTERP

   840 

   841 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp,

   842                                                          Label& zero_continue) {

   843   assert(ProfileInterpreter, "must be profiling interpreter");

   844   movptr(mdp, Address(rbp, frame::interpreter_frame_mdp_offset * wordSize));

   845   testptr(mdp, mdp);

   846   jcc(Assembler::zero, zero_continue);

   847 }

   848 

   849 

   850 // Set the method data pointer for the current bcp.

   851 void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() {

   852   assert(ProfileInterpreter, "must be profiling interpreter");

   853   Label set_mdp;

   854   push(rax);

   855   push(rbx);

   856 

   857   get_method(rbx);

   858   // Test MDO to avoid the call if it is NULL.

   859   movptr(rax, Address(rbx, in_bytes(Method::method_data_offset())));

   860   testptr(rax, rax);

   861   jcc(Assembler::zero, set_mdp);

   862   // rbx: method

   863   // r13: bcp

   864   call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::bcp_to_di), rbx, r13);

   865   // rax: mdi

   866   // mdo is guaranteed to be non-zero here, we checked for it before the call.

   867   movptr(rbx, Address(rbx, in_bytes(Method::method_data_offset())));

   868   addptr(rbx, in_bytes(MethodData::data_offset()));

   869   addptr(rax, rbx);

   870   bind(set_mdp);

   871   movptr(Address(rbp, frame::interpreter_frame_mdp_offset * wordSize), rax);

   872   pop(rbx);

   873   pop(rax);

   874 }

   875 

   876 void InterpreterMacroAssembler::verify_method_data_pointer() {

   877   assert(ProfileInterpreter, "must be profiling interpreter");

   878 #ifdef ASSERT

   879   Label verify_continue;

   880   push(rax);

   881   push(rbx);

   882   push(c_rarg3);

   883   push(c_rarg2);

   884   test_method_data_pointer(c_rarg3, verify_continue); // If mdp is zero, continue

   885   get_method(rbx);

   886 

   887   // If the mdp is valid, it will point to a DataLayout header which is

   888   // consistent with the bcp.  The converse is highly probable also.

   889   load_unsigned_short(c_rarg2,

   890                       Address(c_rarg3, in_bytes(DataLayout::bci_offset())));

   891   addptr(c_rarg2, Address(rbx, Method::const_offset()));

   892   lea(c_rarg2, Address(c_rarg2, ConstMethod::codes_offset()));

   893   cmpptr(c_rarg2, r13);

   894   jcc(Assembler::equal, verify_continue);

   895   // rbx: method

   896   // r13: bcp

   897   // c_rarg3: mdp

   898   call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::verify_mdp),

   899                rbx, r13, c_rarg3);

   900   bind(verify_continue);

   901   pop(c_rarg2);

   902   pop(c_rarg3);

   903   pop(rbx);

   904   pop(rax);

   905 #endif // ASSERT

   906 }

   907 

   908 

   909 void InterpreterMacroAssembler::set_mdp_data_at(Register mdp_in,

   910                                                 int constant,

   911                                                 Register value) {

   912   assert(ProfileInterpreter, "must be profiling interpreter");

   913   Address data(mdp_in, constant);

   914   movptr(data, value);

   915 }

   916 

   917 

   918 void InterpreterMacroAssembler::increment_mdp_data_at(Register mdp_in,

   919                                                       int constant,

   920                                                       bool decrement) {

   921   // Counter address

   922   Address data(mdp_in, constant);

   923 

   924   increment_mdp_data_at(data, decrement);

   925 }

   926 

   927 void InterpreterMacroAssembler::increment_mdp_data_at(Address data,

   928                                                       bool decrement) {

   929   assert(ProfileInterpreter, "must be profiling interpreter");

   930   // %%% this does 64bit counters at best it is wasting space

   931   // at worst it is a rare bug when counters overflow

   932 

   933   if (decrement) {

   934     // Decrement the register.  Set condition codes.

   935     addptr(data, (int32_t) -DataLayout::counter_increment);

   936     // If the decrement causes the counter to overflow, stay negative

   937     Label L;

   938     jcc(Assembler::negative, L);

   939     addptr(data, (int32_t) DataLayout::counter_increment);

   940     bind(L);

   941   } else {

   942     assert(DataLayout::counter_increment == 1,

   943            "flow-free idiom only works with 1");

   944     // Increment the register.  Set carry flag.

   945     addptr(data, DataLayout::counter_increment);

   946     // If the increment causes the counter to overflow, pull back by 1.

   947     sbbptr(data, (int32_t)0);

   948   }

   949 }

   950 

   951 

   952 void InterpreterMacroAssembler::increment_mdp_data_at(Register mdp_in,

   953                                                       Register reg,

   954                                                       int constant,

   955                                                       bool decrement) {

   956   Address data(mdp_in, reg, Address::times_1, constant);

   957 

   958   increment_mdp_data_at(data, decrement);

   959 }

   960 

   961 void InterpreterMacroAssembler::set_mdp_flag_at(Register mdp_in,

   962                                                 int flag_byte_constant) {

   963   assert(ProfileInterpreter, "must be profiling interpreter");

   964   int header_offset = in_bytes(DataLayout::header_offset());

   965   int header_bits = DataLayout::flag_mask_to_header_mask(flag_byte_constant);

   966   // Set the flag

   967   orl(Address(mdp_in, header_offset), header_bits);

   968 }

   969 

   970 

   971 

   972 void InterpreterMacroAssembler::test_mdp_data_at(Register mdp_in,

   973                                                  int offset,

   974                                                  Register value,

   975                                                  Register test_value_out,

   976                                                  Label& not_equal_continue) {

   977   assert(ProfileInterpreter, "must be profiling interpreter");

   978   if (test_value_out == noreg) {

   979     cmpptr(value, Address(mdp_in, offset));

   980   } else {

   981     // Put the test value into a register, so caller can use it:

   982     movptr(test_value_out, Address(mdp_in, offset));

   983     cmpptr(test_value_out, value);

   984   }

   985   jcc(Assembler::notEqual, not_equal_continue);

   986 }

   987 

   988 

   989 void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in,

   990                                                      int offset_of_disp) {

   991   assert(ProfileInterpreter, "must be profiling interpreter");

   992   Address disp_address(mdp_in, offset_of_disp);

   993   addptr(mdp_in, disp_address);

   994   movptr(Address(rbp, frame::interpreter_frame_mdp_offset * wordSize), mdp_in);

   995 }

   996 

   997 

   998 void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in,

   999                                                      Register reg,

  1000                                                      int offset_of_disp) {

  1001   assert(ProfileInterpreter, "must be profiling interpreter");

  1002   Address disp_address(mdp_in, reg, Address::times_1, offset_of_disp);

  1003   addptr(mdp_in, disp_address);

  1004   movptr(Address(rbp, frame::interpreter_frame_mdp_offset * wordSize), mdp_in);

  1005 }

  1006 

  1007 

  1008 void InterpreterMacroAssembler::update_mdp_by_constant(Register mdp_in,

  1009                                                        int constant) {

  1010   assert(ProfileInterpreter, "must be profiling interpreter");

  1011   addptr(mdp_in, constant);

  1012   movptr(Address(rbp, frame::interpreter_frame_mdp_offset * wordSize), mdp_in);

  1013 }

  1014 

  1015 

  1016 void InterpreterMacroAssembler::update_mdp_for_ret(Register return_bci) {

  1017   assert(ProfileInterpreter, "must be profiling interpreter");

  1018   push(return_bci); // save/restore across call_VM

  1019   call_VM(noreg,

  1020           CAST_FROM_FN_PTR(address, InterpreterRuntime::update_mdp_for_ret),

  1021           return_bci);

  1022   pop(return_bci);

  1023 }

  1024 

  1025 

  1026 void InterpreterMacroAssembler::profile_taken_branch(Register mdp,

  1027                                                      Register bumped_count) {

  1028   if (ProfileInterpreter) {

  1029     Label profile_continue;

  1030 

  1031     // If no method data exists, go to profile_continue.

  1032     // Otherwise, assign to mdp

  1033     test_method_data_pointer(mdp, profile_continue);

  1034 

  1035     // We are taking a branch.  Increment the taken count.

  1036     // We inline increment_mdp_data_at to return bumped_count in a register

  1037     //increment_mdp_data_at(mdp, in_bytes(JumpData::taken_offset()));

  1038     Address data(mdp, in_bytes(JumpData::taken_offset()));

  1039     movptr(bumped_count, data);

  1040     assert(DataLayout::counter_increment == 1,

  1041             "flow-free idiom only works with 1");

  1042     addptr(bumped_count, DataLayout::counter_increment);

  1043     sbbptr(bumped_count, 0);

  1044     movptr(data, bumped_count); // Store back out

  1045 

  1046     // The method data pointer needs to be updated to reflect the new target.

  1047     update_mdp_by_offset(mdp, in_bytes(JumpData::displacement_offset()));

  1048     bind(profile_continue);

  1049   }

  1050 }

  1051 

  1052 

  1053 void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp) {

  1054   if (ProfileInterpreter) {

  1055     Label profile_continue;

  1056 

  1057     // If no method data exists, go to profile_continue.

  1058     test_method_data_pointer(mdp, profile_continue);

  1059 

  1060     // We are taking a branch.  Increment the not taken count.

  1061     increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()));

  1062 

  1063     // The method data pointer needs to be updated to correspond to

  1064     // the next bytecode

  1065     update_mdp_by_constant(mdp, in_bytes(BranchData::branch_data_size()));

  1066     bind(profile_continue);

  1067   }

  1068 }

  1069 

  1070 void InterpreterMacroAssembler::profile_call(Register mdp) {

  1071   if (ProfileInterpreter) {

  1072     Label profile_continue;

  1073 

  1074     // If no method data exists, go to profile_continue.

  1075     test_method_data_pointer(mdp, profile_continue);

  1076 

  1077     // We are making a call.  Increment the count.

  1078     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));

  1079 

  1080     // The method data pointer needs to be updated to reflect the new target.

  1081     update_mdp_by_constant(mdp, in_bytes(CounterData::counter_data_size()));

  1082     bind(profile_continue);

  1083   }

  1084 }

  1085 

  1086 

  1087 void InterpreterMacroAssembler::profile_final_call(Register mdp) {

  1088   if (ProfileInterpreter) {

  1089     Label profile_continue;

  1090 

  1091     // If no method data exists, go to profile_continue.

  1092     test_method_data_pointer(mdp, profile_continue);

  1093 

  1094     // We are making a call.  Increment the count.

  1095     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));

  1096 

  1097     // The method data pointer needs to be updated to reflect the new target.

  1098     update_mdp_by_constant(mdp,

  1099                            in_bytes(VirtualCallData::

  1100                                     virtual_call_data_size()));

  1101     bind(profile_continue);

  1102   }

  1103 }

  1104 

  1105 

  1106 void InterpreterMacroAssembler::profile_virtual_call(Register receiver,

  1107                                                      Register mdp,

  1108                                                      Register reg2,

  1109                                                      bool receiver_can_be_null) {

  1110   if (ProfileInterpreter) {

  1111     Label profile_continue;

  1112 

  1113     // If no method data exists, go to profile_continue.

  1114     test_method_data_pointer(mdp, profile_continue);

  1115 

  1116     Label skip_receiver_profile;

  1117     if (receiver_can_be_null) {

  1118       Label not_null;

  1119       testptr(receiver, receiver);

  1120       jccb(Assembler::notZero, not_null);

  1121       // We are making a call.  Increment the count for null receiver.

  1122       increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));

  1123       jmp(skip_receiver_profile);

  1124       bind(not_null);

  1125     }

  1126 

  1127     // Record the receiver type.

  1128     record_klass_in_profile(receiver, mdp, reg2, true);

  1129     bind(skip_receiver_profile);

  1130 

  1131     // The method data pointer needs to be updated to reflect the new target.

  1132     update_mdp_by_constant(mdp,

  1133                            in_bytes(VirtualCallData::

  1134                                     virtual_call_data_size()));