1 /*

     2  * Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved.

     3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

     4  *

     5  * This code is free software; you can redistribute it and/or modify it

     6  * under the terms of the GNU General Public License version 2 only, as

     7  * published by the Free Software Foundation.

     8  *

     9  * This code is distributed in the hope that it will be useful, but WITHOUT

    10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    12  * version 2 for more details (a copy is included in the LICENSE file that

    13  * accompanied this code).

    14  *

    15  * You should have received a copy of the GNU General Public License version

    16  * 2 along with this work; if not, write to the Free Software Foundation,

    17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    18  *

    19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

    20  * or visit www.oracle.com if you need additional information or have any

    21  * questions.

    22  *

    23  */

    24 

    25 #include "precompiled.hpp"

    26 #include "asm/macroAssembler.hpp"

    27 #include "c1/c1_Defs.hpp"

    28 #include "c1/c1_LIRAssembler.hpp"

    29 #include "c1/c1_MacroAssembler.hpp"

    30 #include "c1/c1_Runtime1.hpp"

    31 #include "interpreter/interpreter.hpp"

    32 #include "nativeInst_arm.hpp"

    33 #include "oops/compiledICHolder.hpp"

    34 #include "oops/oop.inline.hpp"

    35 #include "prims/jvmtiExport.hpp"

    36 #include "register_arm.hpp"

    37 #include "runtime/sharedRuntime.hpp"

    38 #include "runtime/signature.hpp"

    39 #include "runtime/vframeArray.hpp"

    40 #include "utilities/align.hpp"

    41 #include "vmreg_arm.inline.hpp"

    42 #if INCLUDE_ALL_GCS

    43 #include "gc/g1/g1SATBCardTableModRefBS.hpp"

    44 #endif

    45 

    46 // Note: Rtemp usage is this file should not impact C2 and should be

    47 // correct as long as it is not implicitly used in lower layers (the

    48 // arm [macro]assembler) and used with care in the other C1 specific

    49 // files.

    50 

    51 // Implementation of StubAssembler

    52 

    53 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, int args_size) {

    54   mov(R0, Rthread);

    55 

    56   int call_offset = set_last_Java_frame(SP, FP, false, Rtemp);

    57 

    58   call(entry);

    59   if (call_offset == -1) { // PC not saved

    60     call_offset = offset();

    61   }

    62   reset_last_Java_frame(Rtemp);

    63 

    64   assert(frame_size() != no_frame_size, "frame must be fixed");

    65   if (_stub_id != Runtime1::forward_exception_id) {

    66     ldr(R3, Address(Rthread, Thread::pending_exception_offset()));

    67   }

    68 

    69   if (oop_result1->is_valid()) {

    70     assert_different_registers(oop_result1, R3, Rtemp);

    71     get_vm_result(oop_result1, Rtemp);

    72   }

    73   if (metadata_result->is_valid()) {

    74     assert_different_registers(metadata_result, R3, Rtemp);

    75     get_vm_result_2(metadata_result, Rtemp);

    76   }

    77 

    78   // Check for pending exception

    79   // unpack_with_exception_in_tls path is taken through

    80   // Runtime1::exception_handler_for_pc

    81   if (_stub_id != Runtime1::forward_exception_id) {

    82     assert(frame_size() != no_frame_size, "cannot directly call forward_exception_id");

    83 #ifdef AARCH64

    84     Label skip;

    85     cbz(R3, skip);

    86     jump(Runtime1::entry_for(Runtime1::forward_exception_id), relocInfo::runtime_call_type, Rtemp);

    87     bind(skip);

    88 #else

    89     cmp(R3, 0);

    90     jump(Runtime1::entry_for(Runtime1::forward_exception_id), relocInfo::runtime_call_type, Rtemp, ne);

    91 #endif // AARCH64

    92   } else {

    93 #ifdef ASSERT

    94     // Should not have pending exception in forward_exception stub

    95     ldr(R3, Address(Rthread, Thread::pending_exception_offset()));

    96     cmp(R3, 0);

    97     breakpoint(ne);

    98 #endif // ASSERT

    99   }

   100   return call_offset;

   101 }

   102 

   103 

   104 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1) {

   105   if (arg1 != R1) {

   106     mov(R1, arg1);

   107   }

   108   return call_RT(oop_result1, metadata_result, entry, 1);

   109 }

   110 

   111 

   112 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2) {

   113   assert(arg1 == R1 && arg2 == R2, "cannot handle otherwise");

   114   return call_RT(oop_result1, metadata_result, entry, 2);

   115 }

   116 

   117 

   118 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2, Register arg3) {

   119   assert(arg1 == R1 && arg2 == R2 && arg3 == R3, "cannot handle otherwise");

   120   return call_RT(oop_result1, metadata_result, entry, 3);

   121 }

   122 

   123 

   124 #define __ sasm->

   125 

   126 // TODO: ARM - does this duplicate RegisterSaver in SharedRuntime?

   127 #ifdef AARCH64

   128 

   129   //

   130   // On AArch64 registers save area has the following layout:

   131   //

   132   // |---------------------|

   133   // | return address (LR) |

   134   // | FP                  |

   135   // |---------------------|

   136   // | D31                 |

   137   // | ...                 |

   138   // | D0                  |

   139   // |---------------------|

   140   // | padding             |

   141   // |---------------------|

   142   // | R28                 |

   143   // | ...                 |

   144   // | R0                  |

   145   // |---------------------| <-- SP

   146   //

   147 

   148 enum RegisterLayout {

   149   number_of_saved_gprs = 29,

   150   number_of_saved_fprs = FloatRegisterImpl::number_of_registers,

   151 

   152   R0_offset  = 0,

   153   D0_offset  = R0_offset + number_of_saved_gprs + 1,

   154   FP_offset  = D0_offset + number_of_saved_fprs,

   155   LR_offset  = FP_offset + 1,

   156 

   157   reg_save_size = LR_offset + 1,

   158 

   159   arg1_offset = reg_save_size * wordSize,

   160   arg2_offset = (reg_save_size + 1) * wordSize

   161 };

   162 

   163 #else

   164 

   165 enum RegisterLayout {

   166   fpu_save_size = pd_nof_fpu_regs_reg_alloc,

   167 #ifndef __SOFTFP__

   168   D0_offset = 0,

   169 #endif

   170   R0_offset = fpu_save_size,

   171   R1_offset,

   172   R2_offset,

   173   R3_offset,

   174   R4_offset,

   175   R5_offset,

   176   R6_offset,

   177 #if (FP_REG_NUM != 7)

   178   R7_offset,

   179 #endif

   180   R8_offset,

   181   R9_offset,

   182   R10_offset,

   183 #if (FP_REG_NUM != 11)

   184   R11_offset,

   185 #endif

   186   R12_offset,

   187   FP_offset,

   188   LR_offset,

   189   reg_save_size,

   190   arg1_offset = reg_save_size * wordSize,

   191   arg2_offset = (reg_save_size + 1) * wordSize

   192 };

   193 

   194 #endif // AARCH64

   195 

   196 static OopMap* generate_oop_map(StubAssembler* sasm, bool save_fpu_registers = HaveVFP) {

   197   sasm->set_frame_size(reg_save_size /* in words */);

   198 

   199   // Record saved value locations in an OopMap.

   200   // Locations are offsets from sp after runtime call.

   201   OopMap* map = new OopMap(VMRegImpl::slots_per_word * reg_save_size, 0);

   202 

   203 #ifdef AARCH64

   204   for (int i = 0; i < number_of_saved_gprs; i++) {

   205     map->set_callee_saved(VMRegImpl::stack2reg((R0_offset + i) * VMRegImpl::slots_per_word), as_Register(i)->as_VMReg());

   206   }

   207   map->set_callee_saved(VMRegImpl::stack2reg(FP_offset * VMRegImpl::slots_per_word), FP->as_VMReg());

   208   map->set_callee_saved(VMRegImpl::stack2reg(LR_offset * VMRegImpl::slots_per_word), LR->as_VMReg());

   209 

   210   if (save_fpu_registers) {

   211     for (int i = 0; i < number_of_saved_fprs; i++) {

   212       map->set_callee_saved(VMRegImpl::stack2reg((D0_offset + i) * VMRegImpl::slots_per_word), as_FloatRegister(i)->as_VMReg());

   213     }

   214   }

   215 #else

   216   int j=0;

   217   for (int i = R0_offset; i < R10_offset; i++) {

   218     if (j == FP_REG_NUM) {

   219       // skip the FP register, saved below

   220       j++;

   221     }

   222     map->set_callee_saved(VMRegImpl::stack2reg(i), as_Register(j)->as_VMReg());

   223     j++;

   224   }

   225   assert(j == R10->encoding(), "must be");

   226 #if (FP_REG_NUM != 11)

   227   // add R11, if not saved as FP

   228   map->set_callee_saved(VMRegImpl::stack2reg(R11_offset), R11->as_VMReg());

   229 #endif

   230   map->set_callee_saved(VMRegImpl::stack2reg(FP_offset), FP->as_VMReg());

   231   map->set_callee_saved(VMRegImpl::stack2reg(LR_offset), LR->as_VMReg());

   232 

   233   if (save_fpu_registers) {

   234     for (int i = 0; i < fpu_save_size; i++) {

   235       map->set_callee_saved(VMRegImpl::stack2reg(i), as_FloatRegister(i)->as_VMReg());

   236     }

   237   }

   238 #endif // AARCH64

   239 

   240   return map;

   241 }

   242 

   243 static OopMap* save_live_registers(StubAssembler* sasm, bool save_fpu_registers = HaveVFP) {

   244   __ block_comment("save_live_registers");

   245   sasm->set_frame_size(reg_save_size /* in words */);

   246 

   247 #ifdef AARCH64

   248   assert((reg_save_size * wordSize) % StackAlignmentInBytes == 0, "SP should be aligned");

   249 

   250   __ raw_push(FP, LR);

   251 

   252   __ sub(SP, SP, (reg_save_size - 2) * wordSize);

   253 

   254   for (int i = 0; i < align_down((int)number_of_saved_gprs, 2); i += 2) {

   255     __ stp(as_Register(i), as_Register(i+1), Address(SP, (R0_offset + i) * wordSize));

   256   }

   257 

   258   if (is_odd(number_of_saved_gprs)) {

   259     int i = number_of_saved_gprs - 1;

   260     __ str(as_Register(i), Address(SP, (R0_offset + i) * wordSize));

   261   }

   262 

   263   if (save_fpu_registers) {

   264     assert (is_even(number_of_saved_fprs), "adjust this code");

   265     for (int i = 0; i < number_of_saved_fprs; i += 2) {

   266       __ stp_d(as_FloatRegister(i), as_FloatRegister(i+1), Address(SP, (D0_offset + i) * wordSize));

   267     }

   268   }

   269 #else

   270   __ push(RegisterSet(FP) | RegisterSet(LR));

   271   __ push(RegisterSet(R0, R6) | RegisterSet(R8, R10) | R12 | altFP_7_11);

   272   if (save_fpu_registers) {

   273     __ fstmdbd(SP, FloatRegisterSet(D0, fpu_save_size / 2), writeback);

   274   } else {

   275     __ sub(SP, SP, fpu_save_size * wordSize);

   276   }

   277 #endif // AARCH64

   278 

   279   return generate_oop_map(sasm, save_fpu_registers);

   280 }

   281 

   282 

   283 static void restore_live_registers(StubAssembler* sasm,

   284                                    bool restore_R0,

   285                                    bool restore_FP_LR,

   286                                    bool do_return,

   287                                    bool restore_fpu_registers = HaveVFP) {

   288   __ block_comment("restore_live_registers");

   289 

   290 #ifdef AARCH64

   291   if (restore_R0) {

   292     __ ldr(R0, Address(SP, R0_offset * wordSize));

   293   }

   294 

   295   assert(is_odd(number_of_saved_gprs), "adjust this code");

   296   for (int i = 1; i < number_of_saved_gprs; i += 2) {

   297     __ ldp(as_Register(i), as_Register(i+1), Address(SP, (R0_offset + i) * wordSize));

   298   }

   299 

   300   if (restore_fpu_registers) {

   301     assert (is_even(number_of_saved_fprs), "adjust this code");

   302     for (int i = 0; i < number_of_saved_fprs; i += 2) {

   303       __ ldp_d(as_FloatRegister(i), as_FloatRegister(i+1), Address(SP, (D0_offset + i) * wordSize));

   304     }

   305   }

   306 

   307   __ add(SP, SP, (reg_save_size - 2) * wordSize);

   308 

   309   if (restore_FP_LR) {

   310     __ raw_pop(FP, LR);

   311     if (do_return) {

   312       __ ret();

   313     }

   314   } else {

   315     assert (!do_return, "return without restoring FP/LR");

   316   }

   317 #else

   318   if (restore_fpu_registers) {

   319     __ fldmiad(SP, FloatRegisterSet(D0, fpu_save_size / 2), writeback);

   320     if (!restore_R0) {

   321       __ add(SP, SP, (R1_offset - fpu_save_size) * wordSize);

   322     }

   323   } else {

   324     __ add(SP, SP, (restore_R0 ? fpu_save_size : R1_offset) * wordSize);

   325   }

   326   __ pop(RegisterSet((restore_R0 ? R0 : R1), R6) | RegisterSet(R8, R10) | R12 | altFP_7_11);

   327   if (restore_FP_LR) {

   328     __ pop(RegisterSet(FP) | RegisterSet(do_return ? PC : LR));

   329   } else {

   330     assert (!do_return, "return without restoring FP/LR");

   331   }

   332 #endif // AARCH64

   333 }

   334 

   335 

   336 static void restore_live_registers_except_R0(StubAssembler* sasm, bool restore_fpu_registers = HaveVFP) {

   337   restore_live_registers(sasm, false, true, true, restore_fpu_registers);

   338 }

   339 

   340 static void restore_live_registers(StubAssembler* sasm, bool restore_fpu_registers = HaveVFP) {

   341   restore_live_registers(sasm, true, true, true, restore_fpu_registers);

   342 }

   343 

   344 #ifndef AARCH64

   345 static void restore_live_registers_except_FP_LR(StubAssembler* sasm, bool restore_fpu_registers = HaveVFP) {

   346   restore_live_registers(sasm, true, false, false, restore_fpu_registers);

   347 }

   348 #endif // !AARCH64

   349 

   350 static void restore_live_registers_without_return(StubAssembler* sasm, bool restore_fpu_registers = HaveVFP) {

   351   restore_live_registers(sasm, true, true, false, restore_fpu_registers);

   352 }

   353 

   354 

   355 void Runtime1::initialize_pd() {

   356 }

   357 

   358 

   359 OopMapSet* Runtime1::generate_exception_throw(StubAssembler* sasm, address target, bool has_argument) {

   360   OopMap* oop_map = save_live_registers(sasm);

   361 

   362   if (has_argument) {

   363     __ ldr(R1, Address(SP, arg1_offset));

   364   }

   365 

   366   int call_offset = __ call_RT(noreg, noreg, target);

   367   OopMapSet* oop_maps = new OopMapSet();

   368   oop_maps->add_gc_map(call_offset, oop_map);

   369 

   370   DEBUG_ONLY(STOP("generate_exception_throw");)  // Should not reach here

   371   return oop_maps;

   372 }

   373 

   374 

   375 static void restore_sp_for_method_handle(StubAssembler* sasm) {

   376   // Restore SP from its saved reg (FP) if the exception PC is a MethodHandle call site.

   377   __ ldr_s32(Rtemp, Address(Rthread, JavaThread::is_method_handle_return_offset()));

   378 #ifdef AARCH64

   379   Label skip;

   380   __ cbz(Rtemp, skip);

   381   __ mov(SP, Rmh_SP_save);

   382   __ bind(skip);

   383 #else

   384   __ cmp(Rtemp, 0);

   385   __ mov(SP, Rmh_SP_save, ne);

   386 #endif // AARCH64

   387 }

   388 

   389 

   390 OopMapSet* Runtime1::generate_handle_exception(StubID id, StubAssembler* sasm) {

   391   __ block_comment("generate_handle_exception");

   392 

   393   bool save_fpu_registers = false;

   394 

   395   // Save registers, if required.

   396   OopMapSet* oop_maps = new OopMapSet();

   397   OopMap* oop_map = NULL;

   398 

   399   switch (id) {

   400   case forward_exception_id: {

   401     save_fpu_registers = HaveVFP;

   402     oop_map = generate_oop_map(sasm);

   403     __ ldr(Rexception_obj, Address(Rthread, Thread::pending_exception_offset()));

   404     __ ldr(Rexception_pc, Address(SP, LR_offset * wordSize));

   405     Register zero = __ zero_register(Rtemp);

   406     __ str(zero, Address(Rthread, Thread::pending_exception_offset()));

   407     break;

   408   }

   409   case handle_exception_id:

   410     save_fpu_registers = HaveVFP;

   411     // fall-through

   412   case handle_exception_nofpu_id:

   413     // At this point all registers MAY be live.

   414     oop_map = save_live_registers(sasm, save_fpu_registers);

   415     break;

   416   case handle_exception_from_callee_id:

   417     // At this point all registers except exception oop (R4/R19) and

   418     // exception pc (R5/R20) are dead.

   419     oop_map = save_live_registers(sasm);  // TODO it's not required to save all registers

   420     break;

   421   default:  ShouldNotReachHere();

   422   }

   423 

   424   __ str(Rexception_obj, Address(Rthread, JavaThread::exception_oop_offset()));

   425   __ str(Rexception_pc, Address(Rthread, JavaThread::exception_pc_offset()));

   426 

   427   __ str(Rexception_pc, Address(SP, LR_offset * wordSize)); // patch throwing pc into return address

   428 

   429   int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, exception_handler_for_pc));

   430   oop_maps->add_gc_map(call_offset, oop_map);

   431 

   432   // Exception handler found

   433   __ str(R0, Address(SP, LR_offset * wordSize)); // patch the return address

   434 

   435   // Restore the registers that were saved at the beginning, remove

   436   // frame and jump to the exception handler.

   437   switch (id) {

   438   case forward_exception_id:

   439   case handle_exception_nofpu_id:

   440   case handle_exception_id:

   441     restore_live_registers(sasm, save_fpu_registers);

   442     // Note: the restore live registers includes the jump to LR (patched to R0)

   443     break;

   444   case handle_exception_from_callee_id:

   445     restore_live_registers_without_return(sasm); // must not jump immediatly to handler

   446     restore_sp_for_method_handle(sasm);

   447     __ ret();

   448     break;

   449   default:  ShouldNotReachHere();

   450   }

   451 

   452   DEBUG_ONLY(STOP("generate_handle_exception");)  // Should not reach here

   453 

   454   return oop_maps;

   455 }

   456 

   457 

   458 void Runtime1::generate_unwind_exception(StubAssembler* sasm) {

   459   // FP no longer used to find the frame start

   460   // on entry, remove_frame() has already been called (restoring FP and LR)

   461 

   462   // search the exception handler address of the caller (using the return address)

   463   __ mov(c_rarg0, Rthread);

   464   __ mov(Rexception_pc, LR);

   465   __ mov(c_rarg1, LR);

   466   __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), c_rarg0, c_rarg1);

   467 

   468   // Exception oop should be still in Rexception_obj and pc in Rexception_pc

   469   // Jump to handler

   470   __ verify_not_null_oop(Rexception_obj);

   471 

   472   // JSR292 extension

   473   restore_sp_for_method_handle(sasm);

   474 

   475   __ jump(R0);

   476 }

   477 

   478 

   479 OopMapSet* Runtime1::generate_patching(StubAssembler* sasm, address target) {

   480   OopMap* oop_map = save_live_registers(sasm);

   481 

   482   // call the runtime patching routine, returns non-zero if nmethod got deopted.

   483   int call_offset = __ call_RT(noreg, noreg, target);

   484   OopMapSet* oop_maps = new OopMapSet();

   485   oop_maps->add_gc_map(call_offset, oop_map);

   486 

   487   DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();

   488   assert(deopt_blob != NULL, "deoptimization blob must have been created");

   489 

   490   __ cmp_32(R0, 0);

   491 

   492 #ifdef AARCH64

   493   Label call_deopt;

   494 

   495   restore_live_registers_without_return(sasm);

   496   __ b(call_deopt, ne);

   497   __ ret();

   498 

   499   __ bind(call_deopt);

   500 #else

   501   restore_live_registers_except_FP_LR(sasm);

   502   __ pop(RegisterSet(FP) | RegisterSet(PC), eq);

   503 

   504   // Deoptimization needed

   505   // TODO: ARM - no need to restore FP & LR because unpack_with_reexecution() stores them back

   506   __ pop(RegisterSet(FP) | RegisterSet(LR));

   507 #endif // AARCH64

   508 

   509   __ jump(deopt_blob->unpack_with_reexecution(), relocInfo::runtime_call_type, Rtemp);

   510 

   511   DEBUG_ONLY(STOP("generate_patching");)  // Should not reach here

   512   return oop_maps;

   513 }

   514 

   515 

   516 OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {

   517   const bool must_gc_arguments = true;

   518   const bool dont_gc_arguments = false;

   519 

   520   OopMapSet* oop_maps = NULL;

   521   bool save_fpu_registers = HaveVFP;

   522 

   523   switch (id) {

   524     case forward_exception_id:

   525       {

   526         oop_maps = generate_handle_exception(id, sasm);

   527         // does not return on ARM

   528       }

   529       break;

   530 

   531 #if INCLUDE_ALL_GCS

   532     case g1_pre_barrier_slow_id:

   533       {

   534         // Input:

   535         // - pre_val pushed on the stack

   536 

   537         __ set_info("g1_pre_barrier_slow_id", dont_gc_arguments);

   538 

   539         // save at least the registers that need saving if the runtime is called

   540 #ifdef AARCH64

   541         __ raw_push(R0, R1);

   542         __ raw_push(R2, R3);

   543         const int nb_saved_regs = 4;

   544 #else // AARCH64

   545         const RegisterSet saved_regs = RegisterSet(R0,R3) | RegisterSet(R12) | RegisterSet(LR);

   546         const int nb_saved_regs = 6;

   547         assert(nb_saved_regs == saved_regs.size(), "fix nb_saved_regs");

   548         __ push(saved_regs);

   549 #endif // AARCH64

   550 

   551         const Register r_pre_val_0  = R0; // must be R0, to be ready for the runtime call

   552         const Register r_index_1    = R1;

   553         const Register r_buffer_2   = R2;

   554 

   555         Address queue_active(Rthread, in_bytes(JavaThread::satb_mark_queue_offset() +

   556                                                SATBMarkQueue::byte_offset_of_active()));

   557         Address queue_index(Rthread, in_bytes(JavaThread::satb_mark_queue_offset() +

   558                                               SATBMarkQueue::byte_offset_of_index()));

   559         Address buffer(Rthread, in_bytes(JavaThread::satb_mark_queue_offset() +

   560                                          SATBMarkQueue::byte_offset_of_buf()));

   561 

   562         Label done;

   563         Label runtime;

   564 

   565         // Is marking still active?

   566         assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");

   567         __ ldrb(R1, queue_active);

   568         __ cbz(R1, done);

   569 

   570         __ ldr(r_index_1, queue_index);

   571         __ ldr(r_pre_val_0, Address(SP, nb_saved_regs*wordSize));

   572         __ ldr(r_buffer_2, buffer);

   573 

   574         __ subs(r_index_1, r_index_1, wordSize);

   575         __ b(runtime, lt);

   576 

   577         __ str(r_index_1, queue_index);

   578         __ str(r_pre_val_0, Address(r_buffer_2, r_index_1));

   579 

   580         __ bind(done);

   581 

   582 #ifdef AARCH64

   583         __ raw_pop(R2, R3);

   584         __ raw_pop(R0, R1);

   585 #else // AARCH64

   586         __ pop(saved_regs);

   587 #endif // AARCH64

   588 

   589         __ ret();

   590 

   591         __ bind(runtime);

   592 

   593         save_live_registers(sasm);

   594 

   595         assert(r_pre_val_0 == c_rarg0, "pre_val should be in R0");

   596         __ mov(c_rarg1, Rthread);

   597         __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), c_rarg0, c_rarg1);

   598 

   599         restore_live_registers_without_return(sasm);

   600 

   601         __ b(done);

   602       }

   603       break;

   604     case g1_post_barrier_slow_id:

   605       {

   606         // Input:

   607         // - store_addr, pushed on the stack

   608 

   609         __ set_info("g1_post_barrier_slow_id", dont_gc_arguments);

   610 

   611         BarrierSet* bs = Universe::heap()->barrier_set();

   612         CardTableModRefBS* ct = barrier_set_cast<CardTableModRefBS>(bs);

   613         Label done;

   614         Label recheck;

   615         Label runtime;

   616 

   617         Address queue_index(Rthread, in_bytes(JavaThread::dirty_card_queue_offset() +

   618                                               DirtyCardQueue::byte_offset_of_index()));

   619         Address buffer(Rthread, in_bytes(JavaThread::dirty_card_queue_offset() +

   620                                          DirtyCardQueue::byte_offset_of_buf()));

   621 

   622         AddressLiteral cardtable((address)ct->byte_map_base, relocInfo::none);

   623         assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");

   624 

   625         // save at least the registers that need saving if the runtime is called

   626 #ifdef AARCH64

   627         __ raw_push(R0, R1);

   628         __ raw_push(R2, R3);

   629         const int nb_saved_regs = 4;

   630 #else // AARCH64

   631         const RegisterSet saved_regs = RegisterSet(R0,R3) | RegisterSet(R12) | RegisterSet(LR);

   632         const int nb_saved_regs = 6;

   633         assert(nb_saved_regs == saved_regs.size(), "fix nb_saved_regs");

   634         __ push(saved_regs);

   635 #endif // AARCH64

   636 

   637         const Register r_card_addr_0 = R0; // must be R0 for the slow case

   638         const Register r_obj_0 = R0;

   639         const Register r_card_base_1 = R1;

   640         const Register r_tmp2 = R2;

   641         const Register r_index_2 = R2;

   642         const Register r_buffer_3 = R3;

   643         const Register tmp1 = Rtemp;

   644 

   645         __ ldr(r_obj_0, Address(SP, nb_saved_regs*wordSize));

   646         // Note: there is a comment in x86 code about not using

   647         // ExternalAddress / lea, due to relocation not working

   648         // properly for that address. Should be OK for arm, where we

   649         // explicitly specify that 'cardtable' has a relocInfo::none

   650         // type.

   651         __ lea(r_card_base_1, cardtable);

   652         __ add(r_card_addr_0, r_card_base_1, AsmOperand(r_obj_0, lsr, CardTableModRefBS::card_shift));

   653 

   654         // first quick check without barrier

   655         __ ldrb(r_tmp2, Address(r_card_addr_0));

   656 

   657         __ cmp(r_tmp2, (int)G1SATBCardTableModRefBS::g1_young_card_val());

   658         __ b(recheck, ne);

   659 

   660         __ bind(done);

   661 

   662 #ifdef AARCH64

   663         __ raw_pop(R2, R3);

   664         __ raw_pop(R0, R1);

   665 #else // AARCH64

   666         __ pop(saved_regs);

   667 #endif // AARCH64

   668 

   669         __ ret();

   670 

   671         __ bind(recheck);

   672 

   673         __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad), tmp1);

   674 

   675         // reload card state after the barrier that ensures the stored oop was visible

   676         __ ldrb(r_tmp2, Address(r_card_addr_0));

   677 

   678         assert(CardTableModRefBS::dirty_card_val() == 0, "adjust this code");

   679         __ cbz(r_tmp2, done);

   680 

   681         // storing region crossing non-NULL, card is clean.

   682         // dirty card and log.

   683 

   684         assert(0 == (int)CardTableModRefBS::dirty_card_val(), "adjust this code");

   685         if (((intptr_t)ct->byte_map_base & 0xff) == 0) {

   686           // Card table is aligned so the lowest byte of the table address base is zero.

   687           __ strb(r_card_base_1, Address(r_card_addr_0));

   688         } else {

   689           __ strb(__ zero_register(r_tmp2), Address(r_card_addr_0));

   690         }

   691 

   692         __ ldr(r_index_2, queue_index);

   693         __ ldr(r_buffer_3, buffer);

   694 

   695         __ subs(r_index_2, r_index_2, wordSize);

   696         __ b(runtime, lt); // go to runtime if now negative

   697 

   698         __ str(r_index_2, queue_index);

   699 

   700         __ str(r_card_addr_0, Address(r_buffer_3, r_index_2));

   701 

   702         __ b(done);

   703 

   704         __ bind(runtime);

   705 

   706         save_live_registers(sasm);

   707 

   708         assert(r_card_addr_0 == c_rarg0, "card_addr should be in R0");

   709         __ mov(c_rarg1, Rthread);

   710         __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), c_rarg0, c_rarg1);

   711 

   712         restore_live_registers_without_return(sasm);

   713 

   714         __ b(done);

   715       }

   716       break;

   717 #endif // INCLUDE_ALL_GCS

   718     case new_instance_id:

   719     case fast_new_instance_id:

   720     case fast_new_instance_init_check_id:

   721       {

   722         const Register result = R0;

   723         const Register klass  = R1;

   724 

   725         if (UseTLAB && FastTLABRefill && id != new_instance_id) {

   726           // We come here when TLAB allocation failed.

   727           // In this case we either refill TLAB or allocate directly from eden.

   728           Label retry_tlab, try_eden, slow_case, slow_case_no_pop;

   729 

   730           // Make sure the class is fully initialized

   731           if (id == fast_new_instance_init_check_id) {

   732             __ ldrb(result, Address(klass, InstanceKlass::init_state_offset()));

   733             __ cmp(result, InstanceKlass::fully_initialized);

   734             __ b(slow_case_no_pop, ne);

   735           }

   736 

   737           // Free some temporary registers

   738           const Register obj_size = R4;

   739           const Register tmp1     = R5;

   740           const Register tmp2     = LR;

   741           const Register obj_end  = Rtemp;

   742 

   743           __ raw_push(R4, R5, LR);

   744 

   745           __ tlab_refill(result, obj_size, tmp1, tmp2, obj_end, try_eden, slow_case);

   746 

   747           __ bind(retry_tlab);

   748           __ ldr_u32(obj_size, Address(klass, Klass::layout_helper_offset()));

   749           __ tlab_allocate(result, obj_end, tmp1, obj_size, slow_case);              // initializes result and obj_end

   750           __ initialize_object(result, obj_end, klass, noreg /* len */, tmp1, tmp2,

   751                                instanceOopDesc::header_size() * HeapWordSize, -1,

   752                                /* is_tlab_allocated */ true);

   753           __ raw_pop_and_ret(R4, R5);

   754 

   755           __ bind(try_eden);

   756           __ ldr_u32(obj_size, Address(klass, Klass::layout_helper_offset()));

   757           __ eden_allocate(result, obj_end, tmp1, tmp2, obj_size, slow_case);        // initializes result and obj_end

   758           __ incr_allocated_bytes(obj_size, tmp2);

   759           __ initialize_object(result, obj_end, klass, noreg /* len */, tmp1, tmp2,

   760                                instanceOopDesc::header_size() * HeapWordSize, -1,

   761                                /* is_tlab_allocated */ false);

   762           __ raw_pop_and_ret(R4, R5);

   763 

   764           __ bind(slow_case);

   765           __ raw_pop(R4, R5, LR);

   766 

   767           __ bind(slow_case_no_pop);

   768         }

   769 

   770         OopMap* map = save_live_registers(sasm);

   771         int call_offset = __ call_RT(result, noreg, CAST_FROM_FN_PTR(address, new_instance), klass);

   772         oop_maps = new OopMapSet();

   773         oop_maps->add_gc_map(call_offset, map);

   774 

   775         // MacroAssembler::StoreStore useless (included in the runtime exit path)

   776 

   777         restore_live_registers_except_R0(sasm);

   778       }

   779       break;

   780 

   781     case counter_overflow_id:

   782       {

   783         OopMap* oop_map = save_live_registers(sasm);

   784         __ ldr(R1, Address(SP, arg1_offset));

   785         __ ldr(R2, Address(SP, arg2_offset));

   786         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, counter_overflow), R1, R2);

   787         oop_maps = new OopMapSet();

   788         oop_maps->add_gc_map(call_offset, oop_map);

   789         restore_live_registers(sasm);

   790       }

   791       break;

   792 

   793     case new_type_array_id:

   794     case new_object_array_id:

   795       {

   796         if (id == new_type_array_id) {

   797           __ set_info("new_type_array", dont_gc_arguments);

   798         } else {

   799           __ set_info("new_object_array", dont_gc_arguments);

   800         }

   801 

   802         const Register result = R0;

   803         const Register klass  = R1;

   804         const Register length = R2;

   805 

   806         if (UseTLAB && FastTLABRefill) {

   807           // We come here when TLAB allocation failed.

   808           // In this case we either refill TLAB or allocate directly from eden.

   809           Label retry_tlab, try_eden, slow_case, slow_case_no_pop;

   810 

   811 #ifdef AARCH64

   812           __ mov_slow(Rtemp, C1_MacroAssembler::max_array_allocation_length);

   813           __ cmp_32(length, Rtemp);

   814 #else

   815           __ cmp_32(length, C1_MacroAssembler::max_array_allocation_length);

   816 #endif // AARCH64

   817           __ b(slow_case_no_pop, hs);

   818 

   819           // Free some temporary registers

   820           const Register arr_size = R4;

   821           const Register tmp1     = R5;

   822           const Register tmp2     = LR;

   823           const Register tmp3     = Rtemp;

   824           const Register obj_end  = tmp3;

   825 

   826           __ raw_push(R4, R5, LR);

   827 

   828           __ tlab_refill(result, arr_size, tmp1, tmp2, tmp3, try_eden, slow_case);

   829 

   830           __ bind(retry_tlab);

   831           // Get the allocation size: round_up((length << (layout_helper & 0xff)) + header_size)

   832           __ ldr_u32(tmp1, Address(klass, Klass::layout_helper_offset()));

   833           __ mov(arr_size, MinObjAlignmentInBytesMask);

   834           __ and_32(tmp2, tmp1, (unsigned int)(Klass::_lh_header_size_mask << Klass::_lh_header_size_shift));

   835 

   836 #ifdef AARCH64

   837           __ lslv_w(tmp3, length, tmp1);

   838           __ add(arr_size, arr_size, tmp3);

   839 #else

   840           __ add(arr_size, arr_size, AsmOperand(length, lsl, tmp1));

   841 #endif // AARCH64

   842 

   843           __ add(arr_size, arr_size, AsmOperand(tmp2, lsr, Klass::_lh_header_size_shift));

   844           __ align_reg(arr_size, arr_size, MinObjAlignmentInBytes);

   845 

   846           // tlab_allocate initializes result and obj_end, and preserves tmp2 which contains header_size

   847           __ tlab_allocate(result, obj_end, tmp1, arr_size, slow_case);

   848 

   849           assert_different_registers(result, obj_end, klass, length, tmp1, tmp2);

   850           __ initialize_header(result, klass, length, tmp1);

   851 

   852           __ add(tmp2, result, AsmOperand(tmp2, lsr, Klass::_lh_header_size_shift));

   853           if (!ZeroTLAB) {

   854             __ initialize_body(tmp2, obj_end, tmp1);

   855           }

   856 

   857           __ membar(MacroAssembler::StoreStore, tmp1);

   858 

   859           __ raw_pop_and_ret(R4, R5);

   860 

   861           __ bind(try_eden);

   862           // Get the allocation size: round_up((length << (layout_helper & 0xff)) + header_size)

   863           __ ldr_u32(tmp1, Address(klass, Klass::layout_helper_offset()));

   864           __ mov(arr_size, MinObjAlignmentInBytesMask);

   865           __ and_32(tmp2, tmp1, (unsigned int)(Klass::_lh_header_size_mask << Klass::_lh_header_size_shift));

   866 

   867 #ifdef AARCH64

   868           __ lslv_w(tmp3, length, tmp1);

   869           __ add(arr_size, arr_size, tmp3);

   870 #else

   871           __ add(arr_size, arr_size, AsmOperand(length, lsl, tmp1));

   872 #endif // AARCH64

   873 

   874           __ add(arr_size, arr_size, AsmOperand(tmp2, lsr, Klass::_lh_header_size_shift));

   875           __ align_reg(arr_size, arr_size, MinObjAlignmentInBytes);

   876 

   877           // eden_allocate destroys tmp2, so reload header_size after allocation

   878           // eden_allocate initializes result and obj_end

   879           __ eden_allocate(result, obj_end, tmp1, tmp2, arr_size, slow_case);

   880           __ incr_allocated_bytes(arr_size, tmp2);

   881           __ ldrb(tmp2, Address(klass, in_bytes(Klass::layout_helper_offset()) +

   882                                        Klass::_lh_header_size_shift / BitsPerByte));

   883           __ initialize_object(result, obj_end, klass, length, tmp1, tmp2, tmp2, -1, /* is_tlab_allocated */ false);

   884           __ raw_pop_and_ret(R4, R5);

   885 

   886           __ bind(slow_case);

   887           __ raw_pop(R4, R5, LR);

   888           __ bind(slow_case_no_pop);

   889         }

   890 

   891         OopMap* map = save_live_registers(sasm);

   892         int call_offset;

   893         if (id == new_type_array_id) {

   894           call_offset = __ call_RT(result, noreg, CAST_FROM_FN_PTR(address, new_type_array), klass, length);

   895         } else {

   896           call_offset = __ call_RT(result, noreg, CAST_FROM_FN_PTR(address, new_object_array), klass, length);

   897         }

   898         oop_maps = new OopMapSet();

   899         oop_maps->add_gc_map(call_offset, map);

   900 

   901         // MacroAssembler::StoreStore useless (included in the runtime exit path)

   902 

   903         restore_live_registers_except_R0(sasm);

   904       }

   905       break;

   906 

   907     case new_multi_array_id:

   908       {

   909         __ set_info("new_multi_array", dont_gc_arguments);

   910 

   911         // R0: klass

   912         // R2: rank

   913         // SP: address of 1st dimension

   914         const Register result = R0;

   915         OopMap* map = save_live_registers(sasm);

   916 

   917         __ mov(R1, R0);

   918         __ add(R3, SP, arg1_offset);

   919         int call_offset = __ call_RT(result, noreg, CAST_FROM_FN_PTR(address, new_multi_array), R1, R2, R3);

   920 

   921         oop_maps = new OopMapSet();

   922         oop_maps->add_gc_map(call_offset, map);

   923 

   924         // MacroAssembler::StoreStore useless (included in the runtime exit path)

   925 

   926         restore_live_registers_except_R0(sasm);

   927       }

   928       break;

   929 

   930     case register_finalizer_id:

   931       {

   932         __ set_info("register_finalizer", dont_gc_arguments);

   933 

   934         // Do not call runtime if JVM_ACC_HAS_FINALIZER flag is not set

   935         __ load_klass(Rtemp, R0);

   936         __ ldr_u32(Rtemp, Address(Rtemp, Klass::access_flags_offset()));

   937 

   938 #ifdef AARCH64

   939         Label L;

   940         __ tbnz(Rtemp, exact_log2(JVM_ACC_HAS_FINALIZER), L);

   941         __ ret();

   942         __ bind(L);

   943 #else

   944         __ tst(Rtemp, JVM_ACC_HAS_FINALIZER);

   945         __ bx(LR, eq);

   946 #endif // AARCH64

   947 

   948         // Call VM

   949         OopMap* map = save_live_registers(sasm);

   950         oop_maps = new OopMapSet();

   951         int call_offset = __ call_RT(noreg, noreg,

   952                                      CAST_FROM_FN_PTR(address, SharedRuntime::register_finalizer), R0);

   953         oop_maps->add_gc_map(call_offset, map);

   954         restore_live_registers(sasm);

   955       }

   956       break;

   957 

   958     case throw_range_check_failed_id:

   959       {

   960         __ set_info("range_check_failed", dont_gc_arguments);

   961         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_range_check_exception), true);

   962       }

   963       break;

   964 

   965     case throw_index_exception_id:

   966       {

   967         __ set_info("index_range_check_failed", dont_gc_arguments);

   968 #ifdef AARCH64

   969         __ NOT_TESTED();

   970 #endif

   971         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_index_exception), true);

   972       }

   973       break;

   974 

   975     case throw_div0_exception_id:

   976       {

   977         __ set_info("throw_div0_exception", dont_gc_arguments);

   978         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_div0_exception), false);

   979       }

   980       break;

   981 

   982     case throw_null_pointer_exception_id:

   983       {

   984         __ set_info("throw_null_pointer_exception", dont_gc_arguments);

   985         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_null_pointer_exception), false);

   986       }

   987       break;

   988 

   989     case handle_exception_nofpu_id:

   990     case handle_exception_id:

   991       {

   992         __ set_info("handle_exception", dont_gc_arguments);

   993         oop_maps = generate_handle_exception(id, sasm);

   994       }

   995       break;

   996 

   997     case handle_exception_from_callee_id:

   998       {

   999         __ set_info("handle_exception_from_callee", dont_gc_arguments);

  1000         oop_maps = generate_handle_exception(id, sasm);

  1001       }

  1002       break;

  1003 

  1004     case unwind_exception_id:

  1005       {

  1006         __ set_info("unwind_exception", dont_gc_arguments);

  1007         generate_unwind_exception(sasm);

  1008       }

  1009       break;

  1010 

  1011     case throw_array_store_exception_id:

  1012       {

  1013         __ set_info("throw_array_store_exception", dont_gc_arguments);

  1014         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_array_store_exception), true);

  1015       }

  1016       break;

  1017 

  1018     case throw_class_cast_exception_id:

  1019       {

  1020         __ set_info("throw_class_cast_exception", dont_gc_arguments);

  1021         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_class_cast_exception), true);

  1022       }

  1023       break;

  1024 

  1025     case throw_incompatible_class_change_error_id:

  1026       {

  1027         __ set_info("throw_incompatible_class_cast_exception", dont_gc_arguments);

  1028 #ifdef AARCH64

  1029         __ NOT_TESTED();

  1030 #endif

  1031         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_incompatible_class_change_error), false);

  1032       }

  1033       break;

  1034 

  1035     case slow_subtype_check_id:

  1036       {

  1037         // (in)  R0 - sub, destroyed,

  1038         // (in)  R1 - super, not changed

  1039         // (out) R0 - result: 1 if check passed, 0 otherwise

  1040         __ raw_push(R2, R3, LR);

  1041 

  1042         // Load an array of secondary_supers

  1043         __ ldr(R2, Address(R0, Klass::secondary_supers_offset()));

  1044         // Length goes to R3

  1045         __ ldr_s32(R3, Address(R2, Array<Klass*>::length_offset_in_bytes()));

  1046         __ add(R2, R2, Array<Klass*>::base_offset_in_bytes());

  1047 

  1048         Label loop, miss;

  1049         __ bind(loop);

  1050         __ cbz(R3, miss);

  1051         __ ldr(LR, Address(R2, wordSize, post_indexed));

  1052         __ sub(R3, R3, 1);

  1053         __ cmp(LR, R1);

  1054         __ b(loop, ne);

  1055 

  1056         // We get here if an equal cache entry is found

  1057         __ str(R1, Address(R0, Klass::secondary_super_cache_offset()));

  1058         __ mov(R0, 1);

  1059         __ raw_pop_and_ret(R2, R3);

  1060 

  1061         // A cache entry not found - return false

  1062         __ bind(miss);

  1063         __ mov(R0, 0);

  1064         __ raw_pop_and_ret(R2, R3);

  1065       }

  1066       break;

  1067 

  1068     case monitorenter_nofpu_id:

  1069       save_fpu_registers = false;

  1070       // fall through

  1071     case monitorenter_id:

  1072       {

  1073         __ set_info("monitorenter", dont_gc_arguments);

  1074         const Register obj  = R1;

  1075         const Register lock = R2;

  1076         OopMap* map = save_live_registers(sasm, save_fpu_registers);

  1077         __ ldr(obj, Address(SP, arg1_offset));

  1078         __ ldr(lock, Address(SP, arg2_offset));

  1079         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorenter), obj, lock);

  1080         oop_maps = new OopMapSet();

  1081         oop_maps->add_gc_map(call_offset, map);

  1082         restore_live_registers(sasm, save_fpu_registers);

  1083       }

  1084       break;

  1085 

  1086     case monitorexit_nofpu_id:

  1087       save_fpu_registers = false;

  1088       // fall through

  1089     case monitorexit_id:

  1090       {

  1091         __ set_info("monitorexit", dont_gc_arguments);

  1092         const Register lock = R1;

  1093         OopMap* map = save_live_registers(sasm, save_fpu_registers);

  1094         __ ldr(lock, Address(SP, arg1_offset));

  1095         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorexit), lock);

  1096         oop_maps = new OopMapSet();

  1097         oop_maps->add_gc_map(call_offset, map);

  1098         restore_live_registers(sasm, save_fpu_registers);

  1099       }

  1100       break;

  1101 

  1102     case deoptimize_id:

  1103       {

  1104         __ set_info("deoptimize", dont_gc_arguments);

  1105         OopMap* oop_map = save_live_registers(sasm);

  1106         const Register trap_request = R1;

  1107         __ ldr(trap_request, Address(SP, arg1_offset));

  1108         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, deoptimize), trap_request);

  1109         oop_maps = new OopMapSet();

  1110         oop_maps->add_gc_map(call_offset, oop_map);

  1111         restore_live_registers_without_return(sasm);

  1112         DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();

  1113         assert(deopt_blob != NULL, "deoptimization blob must have been created");

  1114         __ jump(deopt_blob->unpack_with_reexecution(), relocInfo::runtime_call_type, AARCH64_ONLY(Rtemp) NOT_AARCH64(noreg));

  1115       }

  1116       break;

  1117 

  1118     case access_field_patching_id:

  1119       {

  1120         __ set_info("access_field_patching", dont_gc_arguments);

  1121         oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, access_field_patching));

  1122       }

  1123       break;

  1124 

  1125     case load_klass_patching_id:

  1126       {

  1127         __ set_info("load_klass_patching", dont_gc_arguments);

  1128         oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_klass_patching));

  1129       }

  1130       break;

  1131 

  1132     case load_appendix_patching_id:

  1133       {

  1134         __ set_info("load_appendix_patching", dont_gc_arguments);