1 /*

     2  * Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved.

     3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

     4  *

     5  * This code is free software; you can redistribute it and/or modify it

     6  * under the terms of the GNU General Public License version 2 only, as

     7  * published by the Free Software Foundation.

     8  *

     9  * This code is distributed in the hope that it will be useful, but WITHOUT

    10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    12  * version 2 for more details (a copy is included in the LICENSE file that

    13  * accompanied this code).

    14  *

    15  * You should have received a copy of the GNU General Public License version

    16  * 2 along with this work; if not, write to the Free Software Foundation,

    17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    18  *

    19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

    20  * or visit www.oracle.com if you need additional information or have any

    21  * questions.

    22  *

    23  */

    24 

    25 #include "precompiled.hpp"

    26 #include "asm/assembler.hpp"

    27 #include "assembler_arm.inline.hpp"

    28 #include "interpreter/interpreter.hpp"

    29 #include "nativeInst_arm.hpp"

    30 #include "oops/instanceOop.hpp"

    31 #include "oops/method.hpp"

    32 #include "oops/objArrayKlass.hpp"

    33 #include "oops/oop.inline.hpp"

    34 #include "prims/methodHandles.hpp"

    35 #include "runtime/frame.inline.hpp"

    36 #include "runtime/handles.inline.hpp"

    37 #include "runtime/sharedRuntime.hpp"

    38 #include "runtime/stubCodeGenerator.hpp"

    39 #include "runtime/stubRoutines.hpp"

    40 #ifdef COMPILER2

    41 #include "opto/runtime.hpp"

    42 #endif

    43 

    44 // Declaration and definition of StubGenerator (no .hpp file).

    45 // For a more detailed description of the stub routine structure

    46 // see the comment in stubRoutines.hpp

    47 

    48 #define __ _masm->

    49 

    50 #ifdef PRODUCT

    51 #define BLOCK_COMMENT(str) /* nothing */

    52 #else

    53 #define BLOCK_COMMENT(str) __ block_comment(str)

    54 #endif

    55 

    56 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")

    57 

    58 // -------------------------------------------------------------------------------------------------------------------------

    59 // Stub Code definitions

    60 

    61 // Platform dependent parameters for array copy stubs

    62 

    63 // Note: we have noticed a huge change in behavior on a microbenchmark

    64 // from platform to platform depending on the configuration.

    65 

    66 // Instead of adding a series of command line options (which

    67 // unfortunately have to be done in the shared file and cannot appear

    68 // only in the ARM port), the tested result are hard-coded here in a set

    69 // of options, selected by specifying 'ArmCopyPlatform'

    70 

    71 // Currently, this 'platform' is hardcoded to a value that is a good

    72 // enough trade-off.  However, one can easily modify this file to test

    73 // the hard-coded configurations or create new ones. If the gain is

    74 // significant, we could decide to either add command line options or

    75 // add code to automatically choose a configuration.

    76 

    77 // see comments below for the various configurations created

    78 #define DEFAULT_ARRAYCOPY_CONFIG 0

    79 #define TEGRA2_ARRAYCOPY_CONFIG 1

    80 #define IMX515_ARRAYCOPY_CONFIG 2

    81 

    82 // Hard coded choices (XXX: could be changed to a command line option)

    83 #define ArmCopyPlatform DEFAULT_ARRAYCOPY_CONFIG

    84 

    85 #ifdef AARCH64

    86 #define ArmCopyCacheLineSize 64

    87 #else

    88 #define ArmCopyCacheLineSize 32 // not worth optimizing to 64 according to measured gains

    89 #endif // AARCH64

    90 

    91 // TODO-AARCH64: tune and revise AArch64 arraycopy optimizations

    92 

    93 // configuration for each kind of loop

    94 typedef struct {

    95   int pld_distance;       // prefetch distance (0 => no prefetch, <0: prefetch_before);

    96 #ifndef AARCH64

    97   bool split_ldm;         // if true, split each STM in STMs with fewer registers

    98   bool split_stm;         // if true, split each LTM in LTMs with fewer registers

    99 #endif // !AARCH64

   100 } arraycopy_loop_config;

   101 

   102 // configuration for all loops

   103 typedef struct {

   104   // const char *description;

   105   arraycopy_loop_config forward_aligned;

   106   arraycopy_loop_config backward_aligned;

   107   arraycopy_loop_config forward_shifted;

   108   arraycopy_loop_config backward_shifted;

   109 } arraycopy_platform_config;

   110 

   111 // configured platforms

   112 static arraycopy_platform_config arraycopy_configurations[] = {

   113   // configuration parameters for arraycopy loops

   114 #ifdef AARCH64

   115   {

   116     {-256 }, // forward aligned

   117     {-128 }, // backward aligned

   118     {-256 }, // forward shifted

   119     {-128 }  // backward shifted

   120   }

   121 #else

   122 

   123   // Configurations were chosen based on manual analysis of benchmark

   124   // results, minimizing overhead with respect to best results on the

   125   // different test cases.

   126 

   127   // Prefetch before is always favored since it avoids dirtying the

   128   // cache uselessly for small copies. Code for prefetch after has

   129   // been kept in case the difference is significant for some

   130   // platforms but we might consider dropping it.

   131 

   132   // distance, ldm, stm

   133   {

   134     // default: tradeoff tegra2/imx515/nv-tegra2,

   135     // Notes on benchmarking:

   136     // - not far from optimal configuration on nv-tegra2

   137     // - within 5% of optimal configuration except for backward aligned on IMX

   138     // - up to 40% from optimal configuration for backward shifted and backward align for tegra2

   139     //   but still on par with the operating system copy

   140     {-256, true,  true  }, // forward aligned

   141     {-256, true,  true  }, // backward aligned

   142     {-256, false, false }, // forward shifted

   143     {-256, true,  true  } // backward shifted

   144   },

   145   {

   146     // configuration tuned on tegra2-4.

   147     // Warning: should not be used on nv-tegra2 !

   148     // Notes:

   149     // - prefetch after gives 40% gain on backward copies on tegra2-4,

   150     //   resulting in better number than the operating system

   151     //   copy. However, this can lead to a 300% loss on nv-tegra and has

   152     //   more impact on the cache (fetches futher than what is

   153     //   copied). Use this configuration with care, in case it improves

   154     //   reference benchmarks.

   155     {-256, true,  true  }, // forward aligned

   156     {96,   false, false }, // backward aligned

   157     {-256, false, false }, // forward shifted

   158     {96,   false, false } // backward shifted

   159   },

   160   {

   161     // configuration tuned on imx515

   162     // Notes:

   163     // - smaller prefetch distance is sufficient to get good result and might be more stable

   164     // - refined backward aligned options within 5% of optimal configuration except for

   165     //   tests were the arrays fit in the cache

   166     {-160, false, false }, // forward aligned

   167     {-160, false, false }, // backward aligned

   168     {-160, false, false }, // forward shifted

   169     {-160, true,  true  } // backward shifted

   170   }

   171 #endif // AARCH64

   172 };

   173 

   174 class StubGenerator: public StubCodeGenerator {

   175 

   176 #ifdef PRODUCT

   177 #define inc_counter_np(a,b,c) ((void)0)

   178 #else

   179 #define inc_counter_np(counter, t1, t2) \

   180   BLOCK_COMMENT("inc_counter " #counter); \

   181   __ inc_counter(&counter, t1, t2);

   182 #endif

   183 

   184  private:

   185 

   186   address generate_call_stub(address& return_address) {

   187     StubCodeMark mark(this, "StubRoutines", "call_stub");

   188     address start = __ pc();

   189 

   190 #ifdef AARCH64

   191     const int saved_regs_size = 192;

   192 

   193     __ stp(FP, LR, Address(SP, -saved_regs_size, pre_indexed));

   194     __ mov(FP, SP);

   195 

   196     int sp_offset = 16;

   197     assert(frame::entry_frame_call_wrapper_offset * wordSize == sp_offset, "adjust this code");

   198     __ stp(R0,  ZR,  Address(SP, sp_offset)); sp_offset += 16;

   199 

   200     const int saved_result_and_result_type_offset = sp_offset;

   201     __ stp(R1,  R2,  Address(SP, sp_offset)); sp_offset += 16;

   202     __ stp(R19, R20, Address(SP, sp_offset)); sp_offset += 16;

   203     __ stp(R21, R22, Address(SP, sp_offset)); sp_offset += 16;

   204     __ stp(R23, R24, Address(SP, sp_offset)); sp_offset += 16;

   205     __ stp(R25, R26, Address(SP, sp_offset)); sp_offset += 16;

   206     __ stp(R27, R28, Address(SP, sp_offset)); sp_offset += 16;

   207 

   208     __ stp_d(V8,  V9,  Address(SP, sp_offset)); sp_offset += 16;

   209     __ stp_d(V10, V11, Address(SP, sp_offset)); sp_offset += 16;

   210     __ stp_d(V12, V13, Address(SP, sp_offset)); sp_offset += 16;

   211     __ stp_d(V14, V15, Address(SP, sp_offset)); sp_offset += 16;

   212     assert (sp_offset == saved_regs_size, "adjust this code");

   213 

   214     __ mov(Rmethod, R3);

   215     __ mov(Rthread, R7);

   216     __ reinit_heapbase();

   217 

   218     { // Pass parameters

   219       Label done_parameters, pass_parameters;

   220 

   221       __ mov(Rparams, SP);

   222       __ cbz_w(R6, done_parameters);

   223 

   224       __ sub(Rtemp, SP, R6, ex_uxtw, LogBytesPerWord);

   225       __ align_reg(SP, Rtemp, StackAlignmentInBytes);

   226       __ add(Rparams, SP, R6, ex_uxtw, LogBytesPerWord);

   227 

   228       __ bind(pass_parameters);

   229       __ subs_w(R6, R6, 1);

   230       __ ldr(Rtemp, Address(R5, wordSize, post_indexed));

   231       __ str(Rtemp, Address(Rparams, -wordSize, pre_indexed));

   232       __ b(pass_parameters, ne);

   233 

   234       __ bind(done_parameters);

   235 

   236 #ifdef ASSERT

   237       {

   238         Label L;

   239         __ cmp(SP, Rparams);

   240         __ b(L, eq);

   241         __ stop("SP does not match Rparams");

   242         __ bind(L);

   243       }

   244 #endif

   245     }

   246 

   247     __ mov(Rsender_sp, SP);

   248     __ blr(R4);

   249     return_address = __ pc();

   250 

   251     __ mov(SP, FP);

   252 

   253     __ ldp(R1, R2, Address(SP, saved_result_and_result_type_offset));

   254 

   255     { // Handle return value

   256       Label cont;

   257       __ str(R0, Address(R1));

   258 

   259       __ cmp_w(R2, T_DOUBLE);

   260       __ ccmp_w(R2, T_FLOAT, Assembler::flags_for_condition(eq), ne);

   261       __ b(cont, ne);

   262 

   263       __ str_d(V0, Address(R1));

   264       __ bind(cont);

   265     }

   266 

   267     sp_offset = saved_result_and_result_type_offset + 16;

   268     __ ldp(R19, R20, Address(SP, sp_offset)); sp_offset += 16;

   269     __ ldp(R21, R22, Address(SP, sp_offset)); sp_offset += 16;

   270     __ ldp(R23, R24, Address(SP, sp_offset)); sp_offset += 16;

   271     __ ldp(R25, R26, Address(SP, sp_offset)); sp_offset += 16;

   272     __ ldp(R27, R28, Address(SP, sp_offset)); sp_offset += 16;

   273 

   274     __ ldp_d(V8,  V9,  Address(SP, sp_offset)); sp_offset += 16;

   275     __ ldp_d(V10, V11, Address(SP, sp_offset)); sp_offset += 16;

   276     __ ldp_d(V12, V13, Address(SP, sp_offset)); sp_offset += 16;

   277     __ ldp_d(V14, V15, Address(SP, sp_offset)); sp_offset += 16;

   278     assert (sp_offset == saved_regs_size, "adjust this code");

   279 

   280     __ ldp(FP, LR, Address(SP, saved_regs_size, post_indexed));

   281     __ ret();

   282 

   283 #else // AARCH64

   284 

   285     assert(frame::entry_frame_call_wrapper_offset == 0, "adjust this code");

   286 

   287     __ mov(Rtemp, SP);

   288     __ push(RegisterSet(FP) | RegisterSet(LR));

   289 #ifndef __SOFTFP__

   290     __ fstmdbd(SP, FloatRegisterSet(D8, 8), writeback);

   291 #endif

   292     __ stmdb(SP, RegisterSet(R0, R2) | RegisterSet(R4, R6) | RegisterSet(R8, R10) | altFP_7_11, writeback);

   293     __ mov(Rmethod, R3);

   294     __ ldmia(Rtemp, RegisterSet(R1, R3) | Rthread); // stacked arguments

   295 

   296     // XXX: TODO

   297     // Would be better with respect to native tools if the following

   298     // setting of FP was changed to conform to the native ABI, with FP

   299     // pointing to the saved FP slot (and the corresponding modifications

   300     // for entry_frame_call_wrapper_offset and frame::real_fp).

   301     __ mov(FP, SP);

   302 

   303     {

   304       Label no_parameters, pass_parameters;

   305       __ cmp(R3, 0);

   306       __ b(no_parameters, eq);

   307 

   308       __ bind(pass_parameters);

   309       __ ldr(Rtemp, Address(R2, wordSize, post_indexed)); // Rtemp OK, unused and scratchable

   310       __ subs(R3, R3, 1);

   311       __ push(Rtemp);

   312       __ b(pass_parameters, ne);

   313       __ bind(no_parameters);

   314     }

   315 

   316     __ mov(Rsender_sp, SP);

   317     __ blx(R1);

   318     return_address = __ pc();

   319 

   320     __ add(SP, FP, wordSize); // Skip link to JavaCallWrapper

   321     __ pop(RegisterSet(R2, R3));

   322 #ifndef __ABI_HARD__

   323     __ cmp(R3, T_LONG);

   324     __ cmp(R3, T_DOUBLE, ne);

   325     __ str(R0, Address(R2));

   326     __ str(R1, Address(R2, wordSize), eq);

   327 #else

   328     Label cont, l_float, l_double;

   329 

   330     __ cmp(R3, T_DOUBLE);

   331     __ b(l_double, eq);

   332 

   333     __ cmp(R3, T_FLOAT);

   334     __ b(l_float, eq);

   335 

   336     __ cmp(R3, T_LONG);

   337     __ str(R0, Address(R2));

   338     __ str(R1, Address(R2, wordSize), eq);

   339     __ b(cont);

   340 

   341 

   342     __ bind(l_double);

   343     __ fstd(D0, Address(R2));

   344     __ b(cont);

   345 

   346     __ bind(l_float);

   347     __ fsts(S0, Address(R2));

   348 

   349     __ bind(cont);

   350 #endif

   351 

   352     __ pop(RegisterSet(R4, R6) | RegisterSet(R8, R10) | altFP_7_11);

   353 #ifndef __SOFTFP__

   354     __ fldmiad(SP, FloatRegisterSet(D8, 8), writeback);

   355 #endif

   356     __ pop(RegisterSet(FP) | RegisterSet(PC));

   357 

   358 #endif // AARCH64

   359     return start;

   360   }

   361 

   362 

   363   // (in) Rexception_obj: exception oop

   364   address generate_catch_exception() {

   365     StubCodeMark mark(this, "StubRoutines", "catch_exception");

   366     address start = __ pc();

   367 

   368     __ str(Rexception_obj, Address(Rthread, Thread::pending_exception_offset()));

   369     __ b(StubRoutines::_call_stub_return_address);

   370 

   371     return start;

   372   }

   373 

   374 

   375   // (in) Rexception_pc: return address

   376   address generate_forward_exception() {

   377     StubCodeMark mark(this, "StubRoutines", "forward exception");

   378     address start = __ pc();

   379 

   380     __ mov(c_rarg0, Rthread);

   381     __ mov(c_rarg1, Rexception_pc);

   382     __ call_VM_leaf(CAST_FROM_FN_PTR(address,

   383                          SharedRuntime::exception_handler_for_return_address),

   384                          c_rarg0, c_rarg1);

   385     __ ldr(Rexception_obj, Address(Rthread, Thread::pending_exception_offset()));

   386     const Register Rzero = __ zero_register(Rtemp); // Rtemp OK (cleared by above call)

   387     __ str(Rzero, Address(Rthread, Thread::pending_exception_offset()));

   388 

   389 #ifdef ASSERT

   390     // make sure exception is set

   391     { Label L;

   392       __ cbnz(Rexception_obj, L);

   393       __ stop("StubRoutines::forward exception: no pending exception (2)");

   394       __ bind(L);

   395     }

   396 #endif

   397 

   398     // Verify that there is really a valid exception in RAX.

   399     __ verify_oop(Rexception_obj);

   400 

   401     __ jump(R0); // handler is returned in R0 by runtime function

   402     return start;

   403   }

   404 

   405 

   406 #ifndef AARCH64

   407 

   408   // Integer division shared routine

   409   //   Input:

   410   //     R0  - dividend

   411   //     R2  - divisor

   412   //   Output:

   413   //     R0  - remainder

   414   //     R1  - quotient

   415   //   Destroys:

   416   //     R2

   417   //     LR

   418   address generate_idiv_irem() {

   419     Label positive_arguments, negative_or_zero, call_slow_path;

   420     Register dividend  = R0;

   421     Register divisor   = R2;

   422     Register remainder = R0;

   423     Register quotient  = R1;

   424     Register tmp       = LR;

   425     assert(dividend == remainder, "must be");

   426 

   427     address start = __ pc();

   428 

   429     // Check for special cases: divisor <= 0 or dividend < 0

   430     __ cmp(divisor, 0);

   431     __ orrs(quotient, dividend, divisor, ne);

   432     __ b(negative_or_zero, le);

   433 

   434     __ bind(positive_arguments);

   435     // Save return address on stack to free one extra register

   436     __ push(LR);

   437     // Approximate the mamximum order of the quotient

   438     __ clz(tmp, dividend);

   439     __ clz(quotient, divisor);

   440     __ subs(tmp, quotient, tmp);

   441     __ mov(quotient, 0);

   442     // Jump to the appropriate place in the unrolled loop below

   443     __ ldr(PC, Address(PC, tmp, lsl, 2), pl);

   444     // If divisor is greater than dividend, return immediately

   445     __ pop(PC);

   446 

   447     // Offset table

   448     Label offset_table[32];

   449     int i;

   450     for (i = 0; i <= 31; i++) {

   451       __ emit_address(offset_table[i]);

   452     }

   453 

   454     // Unrolled loop of 32 division steps

   455     for (i = 31; i >= 0; i--) {

   456       __ bind(offset_table[i]);

   457       __ cmp(remainder, AsmOperand(divisor, lsl, i));

   458       __ sub(remainder, remainder, AsmOperand(divisor, lsl, i), hs);

   459       __ add(quotient, quotient, 1 << i, hs);

   460     }

   461     __ pop(PC);

   462 

   463     __ bind(negative_or_zero);

   464     // Find the combination of argument signs and jump to corresponding handler

   465     __ andr(quotient, dividend, 0x80000000, ne);

   466     __ orr(quotient, quotient, AsmOperand(divisor, lsr, 31), ne);

   467     __ add(PC, PC, AsmOperand(quotient, ror, 26), ne);

   468     __ str(LR, Address(Rthread, JavaThread::saved_exception_pc_offset()));

   469 

   470     // The leaf runtime function can destroy R0-R3 and R12 registers which are still alive

   471     RegisterSet saved_registers = RegisterSet(R3) | RegisterSet(R12);

   472 #if R9_IS_SCRATCHED

   473     // Safer to save R9 here since callers may have been written

   474     // assuming R9 survives. This is suboptimal but may not be worth

   475     // revisiting for this slow case.

   476 

   477     // save also R10 for alignment

   478     saved_registers = saved_registers | RegisterSet(R9, R10);

   479 #endif

   480     {

   481       // divisor == 0

   482       FixedSizeCodeBlock zero_divisor(_masm, 8, true);

   483       __ push(saved_registers);

   484       __ mov(R0, Rthread);

   485       __ mov(R1, LR);

   486       __ mov(R2, SharedRuntime::IMPLICIT_DIVIDE_BY_ZERO);

   487       __ b(call_slow_path);

   488     }

   489 

   490     {

   491       // divisor > 0 && dividend < 0

   492       FixedSizeCodeBlock positive_divisor_negative_dividend(_masm, 8, true);

   493       __ push(LR);

   494       __ rsb(dividend, dividend, 0);

   495       __ bl(positive_arguments);

   496       __ rsb(remainder, remainder, 0);

   497       __ rsb(quotient, quotient, 0);

   498       __ pop(PC);

   499     }

   500 

   501     {

   502       // divisor < 0 && dividend > 0

   503       FixedSizeCodeBlock negative_divisor_positive_dividend(_masm, 8, true);

   504       __ push(LR);

   505       __ rsb(divisor, divisor, 0);

   506       __ bl(positive_arguments);

   507       __ rsb(quotient, quotient, 0);

   508       __ pop(PC);

   509     }

   510 

   511     {

   512       // divisor < 0 && dividend < 0

   513       FixedSizeCodeBlock negative_divisor_negative_dividend(_masm, 8, true);

   514       __ push(LR);

   515       __ rsb(dividend, dividend, 0);

   516       __ rsb(divisor, divisor, 0);

   517       __ bl(positive_arguments);

   518       __ rsb(remainder, remainder, 0);

   519       __ pop(PC);

   520     }

   521 

   522     __ bind(call_slow_path);

   523     __ call(CAST_FROM_FN_PTR(address, SharedRuntime::continuation_for_implicit_exception));

   524     __ pop(saved_registers);

   525     __ bx(R0);

   526 

   527     return start;

   528   }

   529 

   530 

   531  // As per atomic.hpp the Atomic read-modify-write operations must be logically implemented as:

   532  //  <fence>; <op>; <membar StoreLoad|StoreStore>

   533  // But for load-linked/store-conditional based systems a fence here simply means

   534  // no load/store can be reordered with respect to the initial load-linked, so we have:

   535  // <membar storeload|loadload> ; load-linked; <op>; store-conditional; <membar storeload|storestore>

   536  // There are no memory actions in <op> so nothing further is needed.

   537  //

   538  // So we define the following for convenience:

   539 #define MEMBAR_ATOMIC_OP_PRE \

   540     MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad|MacroAssembler::LoadLoad)

   541 #define MEMBAR_ATOMIC_OP_POST \

   542     MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad|MacroAssembler::StoreStore)

   543 

   544   // Note: JDK 9 only supports ARMv7+ so we always have ldrexd available even though the

   545   // code below allows for it to be otherwise. The else clause indicates an ARMv5 system

   546   // for which we do not support MP and so membars are not necessary. This ARMv5 code will

   547   // be removed in the future.

   548 

   549   // Support for jint Atomic::add(jint add_value, volatile jint *dest)

   550   //

   551   // Arguments :

   552   //

   553   //      add_value:      R0

   554   //      dest:           R1

   555   //

   556   // Results:

   557   //

   558   //     R0: the new stored in dest

   559   //

   560   // Overwrites:

   561   //

   562   //     R1, R2, R3

   563   //

   564   address generate_atomic_add() {

   565     address start;

   566 

   567     StubCodeMark mark(this, "StubRoutines", "atomic_add");

   568     Label retry;

   569     start = __ pc();

   570     Register addval    = R0;

   571     Register dest      = R1;

   572     Register prev      = R2;

   573     Register ok        = R2;

   574     Register newval    = R3;

   575 

   576     if (VM_Version::supports_ldrex()) {

   577       __ membar(MEMBAR_ATOMIC_OP_PRE, prev);

   578       __ bind(retry);

   579       __ ldrex(newval, Address(dest));

   580       __ add(newval, addval, newval);

   581       __ strex(ok, newval, Address(dest));

   582       __ cmp(ok, 0);

   583       __ b(retry, ne);

   584       __ mov (R0, newval);

   585       __ membar(MEMBAR_ATOMIC_OP_POST, prev);

   586     } else {

   587       __ bind(retry);

   588       __ ldr (prev, Address(dest));

   589       __ add(newval, addval, prev);

   590       __ atomic_cas_bool(prev, newval, dest, 0, noreg/*ignored*/);

   591       __ b(retry, ne);

   592       __ mov (R0, newval);

   593     }

   594     __ bx(LR);

   595 

   596     return start;

   597   }

   598 

   599   // Support for jint Atomic::xchg(jint exchange_value, volatile jint *dest)

   600   //

   601   // Arguments :

   602   //

   603   //      exchange_value: R0

   604   //      dest:           R1

   605   //

   606   // Results:

   607   //

   608   //     R0: the value previously stored in dest

   609   //

   610   // Overwrites:

   611   //

   612   //     R1, R2, R3

   613   //

   614   address generate_atomic_xchg() {

   615     address start;

   616 

   617     StubCodeMark mark(this, "StubRoutines", "atomic_xchg");

   618     start = __ pc();

   619     Register newval    = R0;

   620     Register dest      = R1;

   621     Register prev      = R2;

   622 

   623     Label retry;

   624 

   625     if (VM_Version::supports_ldrex()) {

   626       Register ok=R3;

   627       __ membar(MEMBAR_ATOMIC_OP_PRE, prev);

   628       __ bind(retry);

   629       __ ldrex(prev, Address(dest));

   630       __ strex(ok, newval, Address(dest));

   631       __ cmp(ok, 0);

   632       __ b(retry, ne);

   633       __ mov (R0, prev);

   634       __ membar(MEMBAR_ATOMIC_OP_POST, prev);

   635     } else {

   636       __ bind(retry);

   637       __ ldr (prev, Address(dest));

   638       __ atomic_cas_bool(prev, newval, dest, 0, noreg/*ignored*/);

   639       __ b(retry, ne);

   640       __ mov (R0, prev);

   641     }

   642     __ bx(LR);

   643 

   644     return start;

   645   }

   646 

   647   // Support for jint Atomic::cmpxchg(jint exchange_value, volatile jint *dest, jint compare_value)

   648   //

   649   // Arguments :

   650   //

   651   //      compare_value:  R0

   652   //      exchange_value: R1

   653   //      dest:           R2

   654   //

   655   // Results:

   656   //

   657   //     R0: the value previously stored in dest

   658   //

   659   // Overwrites:

   660   //

   661   //     R0, R1, R2, R3, Rtemp

   662   //

   663   address generate_atomic_cmpxchg() {

   664     address start;

   665 

   666     StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg");

   667     start = __ pc();

   668     Register cmp       = R0;

   669     Register newval    = R1;

   670     Register dest      = R2;

   671     Register temp1     = R3;

   672     Register temp2     = Rtemp; // Rtemp free (native ABI)

   673 

   674     __ membar(MEMBAR_ATOMIC_OP_PRE, temp1);

   675 

   676     // atomic_cas returns previous value in R0

   677     __ atomic_cas(temp1, temp2, cmp, newval, dest, 0);

   678 

   679     __ membar(MEMBAR_ATOMIC_OP_POST, temp1);

   680 

   681     __ bx(LR);

   682 

   683     return start;

   684   }

   685 

   686   // Support for jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong *dest, jlong compare_value)

   687   // reordered before by a wrapper to (jlong compare_value, jlong exchange_value, volatile jlong *dest)

   688   //

   689   // Arguments :

   690   //

   691   //      compare_value:  R1 (High), R0 (Low)

   692   //      exchange_value: R3 (High), R2 (Low)

   693   //      dest:           SP+0

   694   //

   695   // Results:

   696   //

   697   //     R0:R1: the value previously stored in dest

   698   //

   699   // Overwrites:

   700   //

   701   address generate_atomic_cmpxchg_long() {

   702     address start;

   703 

   704     StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg_long");

   705     start = __ pc();

   706     Register cmp_lo      = R0;

   707     Register cmp_hi      = R1;

   708     Register newval_lo   = R2;

   709     Register newval_hi   = R3;

   710     Register addr        = Rtemp;  /* After load from stack */

   711     Register temp_lo     = R4;

   712     Register temp_hi     = R5;

   713     Register temp_result = R8;

   714     assert_different_registers(cmp_lo, newval_lo, temp_lo, addr, temp_result, R7);

   715     assert_different_registers(cmp_hi, newval_hi, temp_hi, addr, temp_result, R7);

   716 

   717     __ membar(MEMBAR_ATOMIC_OP_PRE, Rtemp); // Rtemp free (native ABI)

   718 

   719     // Stack is unaligned, maintain double word alignment by pushing

   720     // odd number of regs.

   721     __ push(RegisterSet(temp_result) | RegisterSet(temp_lo, temp_hi));

   722     __ ldr(addr, Address(SP, 12));

   723 

   724     // atomic_cas64 returns previous value in temp_lo, temp_hi

   725     __ atomic_cas64(temp_lo, temp_hi, temp_result, cmp_lo, cmp_hi,

   726                     newval_lo, newval_hi, addr, 0);

   727     __ mov(R0, temp_lo);

   728     __ mov(R1, temp_hi);

   729 

   730     __ pop(RegisterSet(temp_result) | RegisterSet(temp_lo, temp_hi));

   731 

   732     __ membar(MEMBAR_ATOMIC_OP_POST, Rtemp); // Rtemp free (native ABI)

   733     __ bx(LR);

   734 

   735     return start;

   736   }

   737 

   738   address generate_atomic_load_long() {

   739     address start;

   740 

   741     StubCodeMark mark(this, "StubRoutines", "atomic_load_long");

   742     start = __ pc();

   743     Register result_lo = R0;

   744     Register result_hi = R1;

   745     Register src       = R0;

   746 

   747     if (!os::is_MP()) {

   748       __ ldmia(src, RegisterSet(result_lo, result_hi));

   749       __ bx(LR);

   750     } else if (VM_Version::supports_ldrexd()) {

   751       __ ldrexd(result_lo, Address(src));

   752       __ clrex(); // FIXME: safe to remove?

   753       __ bx(LR);

   754     } else {

   755       __ stop("Atomic load(jlong) unsupported on this platform");

   756       __ bx(LR);

   757     }

   758 

   759     return start;

   760   }

   761 

   762   address generate_atomic_store_long() {

   763     address start;

   764 

   765     StubCodeMark mark(this, "StubRoutines", "atomic_store_long");

   766     start = __ pc();

   767     Register newval_lo = R0;

   768     Register newval_hi = R1;

   769     Register dest      = R2;

   770     Register scratch_lo    = R2;

   771     Register scratch_hi    = R3;  /* After load from stack */

   772     Register result    = R3;

   773 

   774     if (!os::is_MP()) {

   775       __ stmia(dest, RegisterSet(newval_lo, newval_hi));

   776       __ bx(LR);

   777     } else if (VM_Version::supports_ldrexd()) {

   778       __ mov(Rtemp, dest);  // get dest to Rtemp

   779       Label retry;

   780       __ bind(retry);

   781       __ ldrexd(scratch_lo, Address(Rtemp));

   782       __ strexd(result, R0, Address(Rtemp));

   783       __ rsbs(result, result, 1);

   784       __ b(retry, eq);

   785       __ bx(LR);

   786     } else {

   787       __ stop("Atomic store(jlong) unsupported on this platform");

   788       __ bx(LR);

   789     }

   790 

   791     return start;

   792   }

   793 

   794 

   795 #endif // AARCH64

   796 

   797 #ifdef COMPILER2

   798   // Support for uint StubRoutine::Arm::partial_subtype_check( Klass sub, Klass super );

   799   // Arguments :

   800   //

   801   //      ret  : R0, returned

   802   //      icc/xcc: set as R0 (depending on wordSize)

   803   //      sub  : R1, argument, not changed

   804   //      super: R2, argument, not changed

   805   //      raddr: LR, blown by call

   806   address generate_partial_subtype_check() {

   807     __ align(CodeEntryAlignment);

   808     StubCodeMark mark(this, "StubRoutines", "partial_subtype_check");

   809     address start = __ pc();

   810 

   811     // based on SPARC check_klass_subtype_[fast|slow]_path (without CompressedOops)

   812 

   813     // R0 used as tmp_reg (in addition to return reg)

   814     Register sub_klass = R1;

   815     Register super_klass = R2;

   816     Register tmp_reg2 = R3;

   817     Register tmp_reg3 = R4;

   818 #define saved_set tmp_reg2, tmp_reg3

   819 

   820     Label L_loop, L_fail;

   821 

   822     int sc_offset = in_bytes(Klass::secondary_super_cache_offset());

   823 

   824     // fast check should be redundant

   825 

   826     // slow check

   827     {

   828       __ raw_push(saved_set);

   829 

   830       // a couple of useful fields in sub_klass:

   831       int ss_offset = in_bytes(Klass::secondary_supers_offset());

   832 

   833       // Do a linear scan of the secondary super-klass chain.

   834       // This code is rarely used, so simplicity is a virtue here.

   835 

   836       inc_counter_np(SharedRuntime::_partial_subtype_ctr, tmp_reg2, tmp_reg3);

   837 

   838       Register scan_temp = tmp_reg2;

   839       Register count_temp = tmp_reg3;

   840 

   841       // We will consult the secondary-super array.

   842       __ ldr(scan_temp, Address(sub_klass, ss_offset));

   843 

   844       Register search_key = super_klass;

   845 

   846       // Load the array length.

   847       __ ldr_s32(count_temp, Address(scan_temp, Array<Klass*>::length_offset_in_bytes()));

   848       __ add(scan_temp, scan_temp, Array<Klass*>::base_offset_in_bytes());

   849 

   850       __ add(count_temp, count_temp, 1);

   851 

   852       // Top of search loop

   853       __ bind(L_loop);

   854       // Notes:

   855       //  scan_temp starts at the array elements

   856       //  count_temp is 1+size

   857       __ subs(count_temp, count_temp, 1);

   858       __ b(L_fail, eq); // not found in the array

   859 

   860       // Load next super to check

   861       // In the array of super classes elements are pointer sized.

   862       int element_size = wordSize;

   863       __ ldr(R0, Address(scan_temp, element_size, post_indexed));

   864 

   865       // Look for Rsuper_klass on Rsub_klass's secondary super-class-overflow list

   866       __ subs(R0, R0, search_key); // set R0 to 0 on success (and flags to eq)

   867 

   868       // A miss means we are NOT a subtype and need to keep looping

   869       __ b(L_loop, ne);

   870 

   871       // Falling out the bottom means we found a hit; we ARE a subtype

   872 

   873       // Success.  Cache the super we found and proceed in triumph.

   874       __ str(super_klass, Address(sub_klass, sc_offset));

   875 

   876       // Return success

   877       // R0 is already 0 and flags are already set to eq

   878       __ raw_pop(saved_set);

   879       __ ret();

   880 

   881       // Return failure

   882       __ bind(L_fail);

   883 #ifdef AARCH64

   884       // count_temp is 0, can't use ZR here

   885       __ adds(R0, count_temp, 1); // sets the flags

   886 #else

   887       __ movs(R0, 1); // sets the flags

   888 #endif

   889       __ raw_pop(saved_set);

   890       __ ret();

   891     }

   892     return start;

   893   }

   894 #undef saved_set

   895 #endif // COMPILER2

   896 

   897 

   898   //----------------------------------------------------------------------------------------------------

   899   // Non-destructive plausibility checks for oops

   900 

   901   address generate_verify_oop() {

   902     StubCodeMark mark(this, "StubRoutines", "verify_oop");

   903     address start = __ pc();

   904 

   905     // Incoming arguments:

   906     //

   907     // R0: error message (char* )

   908     // R1: address of register save area

   909     // R2: oop to verify

   910     //

   911     // All registers are saved before calling this stub. However, condition flags should be saved here.

   912 

   913     const Register oop   = R2;

   914     const Register klass = R3;

   915     const Register tmp1  = R6;

   916     const Register tmp2  = R8;

   917 

   918     const Register flags     = Rtmp_save0; // R4/R19

   919     const Register ret_addr  = Rtmp_save1; // R5/R20

   920     assert_different_registers(oop, klass, tmp1, tmp2, flags, ret_addr, R7);

   921 

   922     Label exit, error;

   923     InlinedAddress verify_oop_count((address) StubRoutines::verify_oop_count_addr());

   924 

   925 #ifdef AARCH64

   926     __ mrs(flags, Assembler::SysReg_NZCV);

   927 #else

   928     __ mrs(Assembler::CPSR, flags);

   929 #endif // AARCH64

   930 

   931     __ ldr_literal(tmp1, verify_oop_count);

   932     __ ldr_s32(tmp2, Address(tmp1));

   933     __ add(tmp2, tmp2, 1);

   934     __ str_32(tmp2, Address(tmp1));

   935 

   936     // make sure object is 'reasonable'

   937     __ cbz(oop, exit);                           // if obj is NULL it is ok

   938 

   939     // Check if the oop is in the right area of memory

   940     // Note: oop_mask and oop_bits must be updated if the code is saved/reused

   941     const address oop_mask = (address) Universe::verify_oop_mask();

   942     const address oop_bits = (address) Universe::verify_oop_bits();

   943     __ mov_address(tmp1, oop_mask, symbolic_Relocation::oop_mask_reference);

   944     __ andr(tmp2, oop, tmp1);

   945     __ mov_address(tmp1, oop_bits, symbolic_Relocation::oop_bits_reference);

   946     __ cmp(tmp2, tmp1);

   947     __ b(error, ne);

   948 

   949     // make sure klass is 'reasonable'

   950     __ load_klass(klass, oop);                   // get klass

   951     __ cbz(klass, error);                        // if klass is NULL it is broken

   952 

   953     // return if everything seems ok

   954     __ bind(exit);

   955 

   956 #ifdef AARCH64

   957     __ msr(Assembler::SysReg_NZCV, flags);

   958 #else

   959     __ msr(Assembler::CPSR_f, flags);

   960 #endif // AARCH64

   961 

   962     __ ret();

   963 

   964     // handle errors

   965     __ bind(error);

   966 

   967     __ mov(ret_addr, LR);                      // save return address

   968 

   969     // R0: error message

   970     // R1: register save area

   971     __ call(CAST_FROM_FN_PTR(address, MacroAssembler::debug));

   972 

   973     __ mov(LR, ret_addr);

   974     __ b(exit);

   975 

   976     __ bind_literal(verify_oop_count);

   977 

   978     return start;

   979   }

   980 

   981   //----------------------------------------------------------------------------------------------------

   982   // Array copy stubs

   983 

   984   //

   985   //  Generate overlap test for array copy stubs

   986   //

   987   //  Input:

   988   //    R0    -  array1

   989   //    R1    -  array2

   990   //    R2    -  element count, 32-bit int

   991   //

   992   //  input registers are preserved

   993   //

   994   void array_overlap_test(address no_overlap_target, int log2_elem_size, Register tmp1, Register tmp2) {

   995     assert(no_overlap_target != NULL, "must be generated");

   996     array_overlap_test(no_overlap_target, NULL, log2_elem_size, tmp1, tmp2);

   997   }

   998   void array_overlap_test(Label& L_no_overlap, int log2_elem_size, Register tmp1, Register tmp2) {

   999     array_overlap_test(NULL, &L_no_overlap, log2_elem_size, tmp1, tmp2);

  1000   }

  1001   void array_overlap_test(address no_overlap_target, Label* NOLp, int log2_elem_size, Register tmp1, Register tmp2) {

  1002     const Register from       = R0;

  1003     const Register to         = R1;

  1004     const Register count      = R2;

  1005     const Register to_from    = tmp1; // to - from

  1006 #ifndef AARCH64

  1007     const Register byte_count = (log2_elem_size == 0) ? count : tmp2; // count << log2_elem_size

  1008 #endif // AARCH64

  1009     assert_different_registers(from, to, count, tmp1, tmp2);

  1010 

  1011     // no_overlap version works if 'to' lower (unsigned) than 'from'

  1012     // and or 'to' more than (count*size) from 'from'

  1013 

  1014     BLOCK_COMMENT("Array Overlap Test:");

  1015     __ subs(to_from, to, from);

  1016 #ifndef AARCH64

  1017     if (log2_elem_size != 0) {

  1018       __ mov(byte_count, AsmOperand(count, lsl, log2_elem_size));

  1019     }

  1020 #endif // !AARCH64

  1021     if (NOLp == NULL)

  1022       __ b(no_overlap_target,lo);

  1023     else

  1024       __ b((*NOLp), lo);

  1025 #ifdef AARCH64

  1026     __ subs(ZR, to_from, count, ex_sxtw, log2_elem_size);

  1027 #else

  1028     __ cmp(to_from, byte_count);

  1029 #endif // AARCH64

  1030     if (NOLp == NULL)

  1031       __ b(no_overlap_target, ge);

  1032     else

  1033       __ b((*NOLp), ge);

  1034   }

  1035 

  1036 #ifdef AARCH64

  1037   // TODO-AARCH64: revise usages of bulk_* methods (probably ldp`s and stp`s should interlace)

  1038 

  1039   // Loads [from, from + count*wordSize) into regs[0], regs[1], ..., regs[count-1]

  1040   // and increases 'from' by count*wordSize.

  1041   void bulk_load_forward(Register from, const Register regs[], int count) {

  1042     assert (count > 0 && count % 2 == 0, "count must be positive even number");

  1043     int bytes = count * wordSize;

  1044 

  1045     int offset = 0;

  1046     __ ldp(regs[0], regs[1], Address(from, bytes, post_indexed));

  1047     offset += 2*wordSize;

  1048 

  1049     for (int i = 2; i < count; i += 2) {

  1050       __ ldp(regs[i], regs[i+1], Address(from, -bytes + offset));

  1051       offset += 2*wordSize;

  1052     }

  1053 

  1054     assert (offset == bytes, "must be");

  1055   }

  1056 

  1057   // Stores regs[0], regs[1], ..., regs[count-1] to [to, to + count*wordSize)

  1058   // and increases 'to' by count*wordSize.

  1059   void bulk_store_forward(Register to, const Register regs[], int count) {

  1060     assert (count > 0 && count % 2 == 0, "count must be positive even number");

  1061     int bytes = count * wordSize;

  1062 

  1063     int offset = 0;

  1064     __ stp(regs[0], regs[1], Address(to, bytes, post_indexed));

  1065     offset += 2*wordSize;

  1066 

  1067     for (int i = 2; i < count; i += 2) {

  1068       __ stp(regs[i], regs[i+1], Address(to, -bytes + offset));

  1069       offset += 2*wordSize;

  1070     }

  1071 

  1072     assert (offset == bytes, "must be");

  1073   }

  1074 

  1075   // Loads [from - count*wordSize, from) into regs[0], regs[1], ..., regs[count-1]

  1076   // and decreases 'from' by count*wordSize.

  1077   // Note that the word with lowest address goes to regs[0].

  1078   void bulk_load_backward(Register from, const Register regs[], int count) {

  1079     assert (count > 0 && count % 2 == 0, "count must be positive even number");

  1080     int bytes = count * wordSize;

  1081 

  1082     int offset = 0;

  1083 

  1084     for (int i = count - 2; i > 0; i -= 2) {

  1085       offset += 2*wordSize;

  1086       __ ldp(regs[i], regs[i+1], Address(from, -offset));

  1087     }

  1088 

  1089     offset += 2*wordSize;

  1090     __ ldp(regs[0], regs[1], Address(from, -bytes, pre_indexed));

  1091 

  1092     assert (offset == bytes, "must be");

  1093   }

  1094 

  1095   // Stores regs[0], regs[1], ..., regs[count-1] into [to - count*wordSize, to)

  1096   // and decreases 'to' by count*wordSize.

  1097   // Note that regs[0] value goes into the memory with lowest address.

  1098   void bulk_store_backward(Register to, const Register regs[], int count) {

  1099     assert (count > 0 && count % 2 == 0, "count must be positive even number");

  1100     int bytes = count * wordSize;

  1101 

  1102     int offset = 0;

  1103 

  1104     for (int i = count - 2; i > 0; i -= 2) {

  1105       offset += 2*wordSize;

  1106       __ stp(regs[i], regs[i+1], Address(to, -offset));

  1107     }

  1108 

  1109     offset += 2*wordSize;

  1110     __ stp(regs[0], regs[1], Address(to, -bytes, pre_indexed));

  1111 

  1112     assert (offset == bytes, "must be");

  1113   }

  1114 #endif // AARCH64

  1115 

  1116   // TODO-AARCH64: rearrange in-loop prefetches:

  1117   //   probably we should choose between "prefetch-store before or after store", not "before or after load".

  1118   void prefetch(Register from, Register to, int offset, int to_delta = 0) {

  1119     __ prefetch_read(Address(from, offset));

  1120 #ifdef AARCH64

  1121   // Next line commented out to avoid significant loss of performance in memory copy - JDK-8078120

  1122   // __ prfm(pstl1keep, Address(to, offset + to_delta));

  1123 #endif // AARCH64

  1124   }

  1125 

  1126   // Generate the inner loop for forward aligned array copy

  1127   //

  1128   // Arguments

  1129   //      from:      src address, 64 bits  aligned

  1130   //      to:        dst address, wordSize aligned

  1131   //      count:     number of elements (32-bit int)

  1132   //      bytes_per_count: number of bytes for each unit of 'count'

  1133   //

  1134   // Return the minimum initial value for count

  1135   //