/*

 * Copyright (c) 2008, 2018, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "asm/assembler.hpp"

#include "assembler_arm.inline.hpp"

#include "gc/shared/cardTable.hpp"

#include "interpreter/interpreter.hpp"

#include "nativeInst_arm.hpp"

#include "oops/instanceOop.hpp"

#include "oops/method.hpp"

#include "oops/objArrayKlass.hpp"

#include "oops/oop.inline.hpp"

#include "prims/methodHandles.hpp"

#include "runtime/frame.inline.hpp"

#include "runtime/handles.inline.hpp"

#include "runtime/sharedRuntime.hpp"

#include "runtime/stubCodeGenerator.hpp"

#include "runtime/stubRoutines.hpp"

#include "utilities/align.hpp"

#ifdef COMPILER2

#include "opto/runtime.hpp"

#endif

// Declaration and definition of StubGenerator (no .hpp file).

// For a more detailed description of the stub routine structure

// see the comment in stubRoutines.hpp

#define __ _masm->

#ifdef PRODUCT

#define BLOCK_COMMENT(str) /* nothing */

#else

#define BLOCK_COMMENT(str) __ block_comment(str)

#endif

#define BIND(label) bind(label); BLOCK_COMMENT(#label ":")

// -------------------------------------------------------------------------------------------------------------------------

// Stub Code definitions

// Platform dependent parameters for array copy stubs

// Note: we have noticed a huge change in behavior on a microbenchmark

// from platform to platform depending on the configuration.

// Instead of adding a series of command line options (which

// unfortunately have to be done in the shared file and cannot appear

// only in the ARM port), the tested result are hard-coded here in a set

// of options, selected by specifying 'ArmCopyPlatform'

// Currently, this 'platform' is hardcoded to a value that is a good

// enough trade-off.  However, one can easily modify this file to test

// the hard-coded configurations or create new ones. If the gain is

// significant, we could decide to either add command line options or

// add code to automatically choose a configuration.

// see comments below for the various configurations created

#define DEFAULT_ARRAYCOPY_CONFIG 0

#define TEGRA2_ARRAYCOPY_CONFIG 1

#define IMX515_ARRAYCOPY_CONFIG 2

// Hard coded choices (XXX: could be changed to a command line option)

#define ArmCopyPlatform DEFAULT_ARRAYCOPY_CONFIG

#ifdef AARCH64

#define ArmCopyCacheLineSize 64

#else

#define ArmCopyCacheLineSize 32 // not worth optimizing to 64 according to measured gains

#endif // AARCH64

// TODO-AARCH64: tune and revise AArch64 arraycopy optimizations

// configuration for each kind of loop

typedef struct {

  int pld_distance;       // prefetch distance (0 => no prefetch, <0: prefetch_before);

#ifndef AARCH64

  bool split_ldm;         // if true, split each STM in STMs with fewer registers

  bool split_stm;         // if true, split each LTM in LTMs with fewer registers

#endif // !AARCH64

} arraycopy_loop_config;

// configuration for all loops

typedef struct {

  // const char *description;

  arraycopy_loop_config forward_aligned;

  arraycopy_loop_config backward_aligned;

  arraycopy_loop_config forward_shifted;

  arraycopy_loop_config backward_shifted;

} arraycopy_platform_config;

// configured platforms

static arraycopy_platform_config arraycopy_configurations[] = {

  // configuration parameters for arraycopy loops

#ifdef AARCH64

  {

    {-256 }, // forward aligned

    {-128 }, // backward aligned

    {-256 }, // forward shifted

    {-128 }  // backward shifted

  }

#else

  // Configurations were chosen based on manual analysis of benchmark

  // results, minimizing overhead with respect to best results on the

  // different test cases.

  // Prefetch before is always favored since it avoids dirtying the

  // cache uselessly for small copies. Code for prefetch after has

  // been kept in case the difference is significant for some

  // platforms but we might consider dropping it.

  // distance, ldm, stm

  {

    // default: tradeoff tegra2/imx515/nv-tegra2,

    // Notes on benchmarking:

    // - not far from optimal configuration on nv-tegra2

    // - within 5% of optimal configuration except for backward aligned on IMX

    // - up to 40% from optimal configuration for backward shifted and backward align for tegra2

    //   but still on par with the operating system copy

    {-256, true,  true  }, // forward aligned

    {-256, true,  true  }, // backward aligned

    {-256, false, false }, // forward shifted

    {-256, true,  true  } // backward shifted

  },

  {

    // configuration tuned on tegra2-4.

    // Warning: should not be used on nv-tegra2 !

    // Notes:

    // - prefetch after gives 40% gain on backward copies on tegra2-4,

    //   resulting in better number than the operating system

    //   copy. However, this can lead to a 300% loss on nv-tegra and has

    //   more impact on the cache (fetches futher than what is

    //   copied). Use this configuration with care, in case it improves

    //   reference benchmarks.

    {-256, true,  true  }, // forward aligned

    {96,   false, false }, // backward aligned

    {-256, false, false }, // forward shifted

    {96,   false, false } // backward shifted

  },

  {

    // configuration tuned on imx515

    // Notes:

    // - smaller prefetch distance is sufficient to get good result and might be more stable

    // - refined backward aligned options within 5% of optimal configuration except for

    //   tests were the arrays fit in the cache

    {-160, false, false }, // forward aligned

    {-160, false, false }, // backward aligned

    {-160, false, false }, // forward shifted

    {-160, true,  true  } // backward shifted

  }

#endif // AARCH64

};

class StubGenerator: public StubCodeGenerator {

#ifdef PRODUCT

#define inc_counter_np(a,b,c) ((void)0)

#else

#define inc_counter_np(counter, t1, t2) \

  BLOCK_COMMENT("inc_counter " #counter); \

  __ inc_counter(&counter, t1, t2);

#endif

 private:

  address generate_call_stub(address& return_address) {

    StubCodeMark mark(this, "StubRoutines", "call_stub");

    address start = __ pc();

#ifdef AARCH64

    const int saved_regs_size = 192;

    __ stp(FP, LR, Address(SP, -saved_regs_size, pre_indexed));

    __ mov(FP, SP);

    int sp_offset = 16;

    assert(frame::entry_frame_call_wrapper_offset * wordSize == sp_offset, "adjust this code");

    __ stp(R0,  ZR,  Address(SP, sp_offset)); sp_offset += 16;

    const int saved_result_and_result_type_offset = sp_offset;

    __ stp(R1,  R2,  Address(SP, sp_offset)); sp_offset += 16;

    __ stp(R19, R20, Address(SP, sp_offset)); sp_offset += 16;

    __ stp(R21, R22, Address(SP, sp_offset)); sp_offset += 16;

    __ stp(R23, R24, Address(SP, sp_offset)); sp_offset += 16;

    __ stp(R25, R26, Address(SP, sp_offset)); sp_offset += 16;

    __ stp(R27, R28, Address(SP, sp_offset)); sp_offset += 16;

    __ stp_d(V8,  V9,  Address(SP, sp_offset)); sp_offset += 16;

    __ stp_d(V10, V11, Address(SP, sp_offset)); sp_offset += 16;

    __ stp_d(V12, V13, Address(SP, sp_offset)); sp_offset += 16;

    __ stp_d(V14, V15, Address(SP, sp_offset)); sp_offset += 16;

    assert (sp_offset == saved_regs_size, "adjust this code");

    __ mov(Rmethod, R3);

    __ mov(Rthread, R7);

    __ reinit_heapbase();

    { // Pass parameters

      Label done_parameters, pass_parameters;

      __ mov(Rparams, SP);

      __ cbz_w(R6, done_parameters);

      __ sub(Rtemp, SP, R6, ex_uxtw, LogBytesPerWord);

      __ align_reg(SP, Rtemp, StackAlignmentInBytes);

      __ add(Rparams, SP, R6, ex_uxtw, LogBytesPerWord);

      __ bind(pass_parameters);

      __ subs_w(R6, R6, 1);

      __ ldr(Rtemp, Address(R5, wordSize, post_indexed));

      __ str(Rtemp, Address(Rparams, -wordSize, pre_indexed));

      __ b(pass_parameters, ne);

      __ bind(done_parameters);

#ifdef ASSERT

      {

        Label L;

        __ cmp(SP, Rparams);

        __ b(L, eq);

        __ stop("SP does not match Rparams");

        __ bind(L);

      }

#endif

    }

    __ mov(Rsender_sp, SP);

    __ blr(R4);

    return_address = __ pc();

    __ mov(SP, FP);

    __ ldp(R1, R2, Address(SP, saved_result_and_result_type_offset));

    { // Handle return value

      Label cont;

      __ str(R0, Address(R1));

      __ cmp_w(R2, T_DOUBLE);

      __ ccmp_w(R2, T_FLOAT, Assembler::flags_for_condition(eq), ne);

      __ b(cont, ne);

      __ str_d(V0, Address(R1));

      __ bind(cont);

    }

    sp_offset = saved_result_and_result_type_offset + 16;

    __ ldp(R19, R20, Address(SP, sp_offset)); sp_offset += 16;

    __ ldp(R21, R22, Address(SP, sp_offset)); sp_offset += 16;

    __ ldp(R23, R24, Address(SP, sp_offset)); sp_offset += 16;

    __ ldp(R25, R26, Address(SP, sp_offset)); sp_offset += 16;

    __ ldp(R27, R28, Address(SP, sp_offset)); sp_offset += 16;

    __ ldp_d(V8,  V9,  Address(SP, sp_offset)); sp_offset += 16;

    __ ldp_d(V10, V11, Address(SP, sp_offset)); sp_offset += 16;

    __ ldp_d(V12, V13, Address(SP, sp_offset)); sp_offset += 16;

    __ ldp_d(V14, V15, Address(SP, sp_offset)); sp_offset += 16;

    assert (sp_offset == saved_regs_size, "adjust this code");

    __ ldp(FP, LR, Address(SP, saved_regs_size, post_indexed));

    __ ret();

#else // AARCH64

    assert(frame::entry_frame_call_wrapper_offset == 0, "adjust this code");

    __ mov(Rtemp, SP);

    __ push(RegisterSet(FP) | RegisterSet(LR));

#ifndef __SOFTFP__

    __ fstmdbd(SP, FloatRegisterSet(D8, 8), writeback);

#endif

    __ stmdb(SP, RegisterSet(R0, R2) | RegisterSet(R4, R6) | RegisterSet(R8, R10) | altFP_7_11, writeback);

    __ mov(Rmethod, R3);

    __ ldmia(Rtemp, RegisterSet(R1, R3) | Rthread); // stacked arguments

    // XXX: TODO

    // Would be better with respect to native tools if the following

    // setting of FP was changed to conform to the native ABI, with FP

    // pointing to the saved FP slot (and the corresponding modifications

    // for entry_frame_call_wrapper_offset and frame::real_fp).

    __ mov(FP, SP);

    {

      Label no_parameters, pass_parameters;

      __ cmp(R3, 0);

      __ b(no_parameters, eq);

      __ bind(pass_parameters);

      __ ldr(Rtemp, Address(R2, wordSize, post_indexed)); // Rtemp OK, unused and scratchable

      __ subs(R3, R3, 1);

      __ push(Rtemp);

      __ b(pass_parameters, ne);

      __ bind(no_parameters);

    }

    __ mov(Rsender_sp, SP);

    __ blx(R1);

    return_address = __ pc();

    __ add(SP, FP, wordSize); // Skip link to JavaCallWrapper

    __ pop(RegisterSet(R2, R3));

#ifndef __ABI_HARD__

    __ cmp(R3, T_LONG);

    __ cmp(R3, T_DOUBLE, ne);

    __ str(R0, Address(R2));

    __ str(R1, Address(R2, wordSize), eq);

#else

    Label cont, l_float, l_double;

    __ cmp(R3, T_DOUBLE);

    __ b(l_double, eq);

    __ cmp(R3, T_FLOAT);

    __ b(l_float, eq);

    __ cmp(R3, T_LONG);

    __ str(R0, Address(R2));

    __ str(R1, Address(R2, wordSize), eq);

    __ b(cont);

    __ bind(l_double);

    __ fstd(D0, Address(R2));

    __ b(cont);

    __ bind(l_float);

    __ fsts(S0, Address(R2));

    __ bind(cont);

#endif

    __ pop(RegisterSet(R4, R6) | RegisterSet(R8, R10) | altFP_7_11);

#ifndef __SOFTFP__

    __ fldmiad(SP, FloatRegisterSet(D8, 8), writeback);

#endif

    __ pop(RegisterSet(FP) | RegisterSet(PC));

#endif // AARCH64

    return start;

  }

  // (in) Rexception_obj: exception oop

  address generate_catch_exception() {

    StubCodeMark mark(this, "StubRoutines", "catch_exception");

    address start = __ pc();

    __ str(Rexception_obj, Address(Rthread, Thread::pending_exception_offset()));

    __ b(StubRoutines::_call_stub_return_address);

    return start;

  }

  // (in) Rexception_pc: return address

  address generate_forward_exception() {

    StubCodeMark mark(this, "StubRoutines", "forward exception");

    address start = __ pc();

    __ mov(c_rarg0, Rthread);

    __ mov(c_rarg1, Rexception_pc);

    __ call_VM_leaf(CAST_FROM_FN_PTR(address,

                         SharedRuntime::exception_handler_for_return_address),

                         c_rarg0, c_rarg1);

    __ ldr(Rexception_obj, Address(Rthread, Thread::pending_exception_offset()));

    const Register Rzero = __ zero_register(Rtemp); // Rtemp OK (cleared by above call)

    __ str(Rzero, Address(Rthread, Thread::pending_exception_offset()));

#ifdef ASSERT

    // make sure exception is set

    { Label L;

      __ cbnz(Rexception_obj, L);

      __ stop("StubRoutines::forward exception: no pending exception (2)");

      __ bind(L);

    }

#endif

    // Verify that there is really a valid exception in RAX.

    __ verify_oop(Rexception_obj);

    __ jump(R0); // handler is returned in R0 by runtime function

    return start;

  }

#ifndef AARCH64

  // Integer division shared routine

  //   Input:

  //     R0  - dividend

  //     R2  - divisor

  //   Output:

  //     R0  - remainder

  //     R1  - quotient

  //   Destroys:

  //     R2

  //     LR

  address generate_idiv_irem() {

    Label positive_arguments, negative_or_zero, call_slow_path;

    Register dividend  = R0;

    Register divisor   = R2;

    Register remainder = R0;

    Register quotient  = R1;

    Register tmp       = LR;

    assert(dividend == remainder, "must be");

    address start = __ pc();

    // Check for special cases: divisor <= 0 or dividend < 0

    __ cmp(divisor, 0);

    __ orrs(quotient, dividend, divisor, ne);

    __ b(negative_or_zero, le);

    __ bind(positive_arguments);

    // Save return address on stack to free one extra register

    __ push(LR);

    // Approximate the mamximum order of the quotient

    __ clz(tmp, dividend);

    __ clz(quotient, divisor);

    __ subs(tmp, quotient, tmp);

    __ mov(quotient, 0);

    // Jump to the appropriate place in the unrolled loop below

    __ ldr(PC, Address(PC, tmp, lsl, 2), pl);

    // If divisor is greater than dividend, return immediately

    __ pop(PC);

    // Offset table

    Label offset_table[32];

    int i;

    for (i = 0; i <= 31; i++) {

      __ emit_address(offset_table[i]);

    }

    // Unrolled loop of 32 division steps

    for (i = 31; i >= 0; i--) {

      __ bind(offset_table[i]);

      __ cmp(remainder, AsmOperand(divisor, lsl, i));

      __ sub(remainder, remainder, AsmOperand(divisor, lsl, i), hs);

      __ add(quotient, quotient, 1 << i, hs);

    }

    __ pop(PC);

    __ bind(negative_or_zero);

    // Find the combination of argument signs and jump to corresponding handler

    __ andr(quotient, dividend, 0x80000000, ne);

    __ orr(quotient, quotient, AsmOperand(divisor, lsr, 31), ne);

    __ add(PC, PC, AsmOperand(quotient, ror, 26), ne);

    __ str(LR, Address(Rthread, JavaThread::saved_exception_pc_offset()));

    // The leaf runtime function can destroy R0-R3 and R12 registers which are still alive

    RegisterSet saved_registers = RegisterSet(R3) | RegisterSet(R12);

#if R9_IS_SCRATCHED

    // Safer to save R9 here since callers may have been written

    // assuming R9 survives. This is suboptimal but may not be worth

    // revisiting for this slow case.

    // save also R10 for alignment

    saved_registers = saved_registers | RegisterSet(R9, R10);

#endif

    {

      // divisor == 0

      FixedSizeCodeBlock zero_divisor(_masm, 8, true);

      __ push(saved_registers);

      __ mov(R0, Rthread);

      __ mov(R1, LR);

      __ mov(R2, SharedRuntime::IMPLICIT_DIVIDE_BY_ZERO);

      __ b(call_slow_path);

    }

    {

      // divisor > 0 && dividend < 0

      FixedSizeCodeBlock positive_divisor_negative_dividend(_masm, 8, true);

      __ push(LR);

      __ rsb(dividend, dividend, 0);

      __ bl(positive_arguments);

      __ rsb(remainder, remainder, 0);

      __ rsb(quotient, quotient, 0);

      __ pop(PC);

    }

    {

      // divisor < 0 && dividend > 0

      FixedSizeCodeBlock negative_divisor_positive_dividend(_masm, 8, true);

      __ push(LR);

      __ rsb(divisor, divisor, 0);

      __ bl(positive_arguments);

      __ rsb(quotient, quotient, 0);

      __ pop(PC);

    }

    {

      // divisor < 0 && dividend < 0

      FixedSizeCodeBlock negative_divisor_negative_dividend(_masm, 8, true);

      __ push(LR);

      __ rsb(dividend, dividend, 0);

      __ rsb(divisor, divisor, 0);

      __ bl(positive_arguments);

      __ rsb(remainder, remainder, 0);

      __ pop(PC);

    }

    __ bind(call_slow_path);

    __ call(CAST_FROM_FN_PTR(address, SharedRuntime::continuation_for_implicit_exception));

    __ pop(saved_registers);

    __ bx(R0);