/*

 * Copyright (c) 2008, 2018, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "jvm.h"

#include "gc/shared/barrierSet.inline.hpp"

#include "gc/shared/cardTable.hpp"

#include "gc/shared/collectedHeap.hpp"

#include "interp_masm_arm.hpp"

#include "interpreter/interpreter.hpp"

#include "interpreter/interpreterRuntime.hpp"

#include "logging/log.hpp"

#include "oops/arrayOop.hpp"

#include "oops/markOop.hpp"

#include "oops/method.hpp"

#include "oops/methodData.hpp"

#include "prims/jvmtiExport.hpp"

#include "prims/jvmtiThreadState.hpp"

#include "runtime/basicLock.hpp"

#include "runtime/biasedLocking.hpp"

#include "runtime/sharedRuntime.hpp"

#if INCLUDE_ALL_GCS

#include "gc/g1/g1BarrierSet.hpp"

#include "gc/g1/g1CollectedHeap.inline.hpp"

#include "gc/g1/heapRegion.hpp"

#endif // INCLUDE_ALL_GCS

//--------------------------------------------------------------------

// Implementation of InterpreterMacroAssembler

InterpreterMacroAssembler::InterpreterMacroAssembler(CodeBuffer* code) : MacroAssembler(code) {

}

void InterpreterMacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {

#if defined(ASSERT) && !defined(AARCH64)

  // Ensure that last_sp is not filled.

  { Label L;

    ldr(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));

    cbz(Rtemp, L);

    stop("InterpreterMacroAssembler::call_VM_helper: last_sp != NULL");

    bind(L);

  }

#endif // ASSERT && !AARCH64

  // Rbcp must be saved/restored since it may change due to GC.

  save_bcp();

#ifdef AARCH64

  check_no_cached_stack_top(Rtemp);

  save_stack_top();

  check_extended_sp(Rtemp);

  cut_sp_before_call();

#endif // AARCH64

  // super call

  MacroAssembler::call_VM_helper(oop_result, entry_point, number_of_arguments, check_exceptions);

#ifdef AARCH64

  // Restore SP to extended SP

  restore_sp_after_call(Rtemp);

  check_stack_top();

  clear_cached_stack_top();

#endif // AARCH64

  // Restore interpreter specific registers.

  restore_bcp();

  restore_method();

}

void InterpreterMacroAssembler::jump_to_entry(address entry) {

  assert(entry, "Entry must have been generated by now");

  b(entry);

}

void InterpreterMacroAssembler::check_and_handle_popframe() {

  if (can_pop_frame()) {

    Label L;

    const Register popframe_cond = R2_tmp;

    // Initiate popframe handling only if it is not already being processed.  If the flag

    // has the popframe_processing bit set, it means that this code is called *during* popframe

    // handling - we don't want to reenter.

    ldr_s32(popframe_cond, Address(Rthread, JavaThread::popframe_condition_offset()));

    tbz(popframe_cond, exact_log2(JavaThread::popframe_pending_bit), L);

    tbnz(popframe_cond, exact_log2(JavaThread::popframe_processing_bit), L);

    // Call Interpreter::remove_activation_preserving_args_entry() to get the

    // address of the same-named entrypoint in the generated interpreter code.

    call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_preserving_args_entry));

    // Call indirectly to avoid generation ordering problem.

    jump(R0);

    bind(L);

  }

}

// Blows R2, Rtemp. Sets TOS cached value.

void InterpreterMacroAssembler::load_earlyret_value(TosState state) {

  const Register thread_state = R2_tmp;

  ldr(thread_state, Address(Rthread, JavaThread::jvmti_thread_state_offset()));

  const Address tos_addr(thread_state, JvmtiThreadState::earlyret_tos_offset());

  const Address oop_addr(thread_state, JvmtiThreadState::earlyret_oop_offset());

  const Address val_addr(thread_state, JvmtiThreadState::earlyret_value_offset());

#ifndef AARCH64

  const Address val_addr_hi(thread_state, JvmtiThreadState::earlyret_value_offset()

                             + in_ByteSize(wordSize));

#endif // !AARCH64

  Register zero = zero_register(Rtemp);

  switch (state) {

    case atos: ldr(R0_tos, oop_addr);

               str(zero, oop_addr);

               interp_verify_oop(R0_tos, state, __FILE__, __LINE__);

               break;

#ifdef AARCH64

    case ltos: ldr(R0_tos, val_addr);              break;

#else

    case ltos: ldr(R1_tos_hi, val_addr_hi);        // fall through

#endif // AARCH64

    case btos:                                     // fall through

    case ztos:                                     // fall through

    case ctos:                                     // fall through

    case stos:                                     // fall through

    case itos: ldr_s32(R0_tos, val_addr);          break;

#ifdef __SOFTFP__

    case dtos: ldr(R1_tos_hi, val_addr_hi);        // fall through

    case ftos: ldr(R0_tos, val_addr);              break;

#else

    case ftos: ldr_float (S0_tos, val_addr);       break;

    case dtos: ldr_double(D0_tos, val_addr);       break;

#endif // __SOFTFP__

    case vtos: /* nothing to do */                 break;

    default  : ShouldNotReachHere();

  }

  // Clean up tos value in the thread object

  str(zero, val_addr);

#ifndef AARCH64

  str(zero, val_addr_hi);

#endif // !AARCH64

  mov(Rtemp, (int) ilgl);

  str_32(Rtemp, tos_addr);

}

// Blows R2, Rtemp.

void InterpreterMacroAssembler::check_and_handle_earlyret() {

  if (can_force_early_return()) {

    Label L;

    const Register thread_state = R2_tmp;

    ldr(thread_state, Address(Rthread, JavaThread::jvmti_thread_state_offset()));

    cbz(thread_state, L); // if (thread->jvmti_thread_state() == NULL) exit;

    // Initiate earlyret handling only if it is not already being processed.

    // If the flag has the earlyret_processing bit set, it means that this code

    // is called *during* earlyret handling - we don't want to reenter.

    ldr_s32(Rtemp, Address(thread_state, JvmtiThreadState::earlyret_state_offset()));

    cmp(Rtemp, JvmtiThreadState::earlyret_pending);

    b(L, ne);

    // Call Interpreter::remove_activation_early_entry() to get the address of the

    // same-named entrypoint in the generated interpreter code.

    ldr_s32(R0, Address(thread_state, JvmtiThreadState::earlyret_tos_offset()));

    call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_early_entry), R0);

    jump(R0);

    bind(L);

  }

}

// Sets reg. Blows Rtemp.

void InterpreterMacroAssembler::get_unsigned_2_byte_index_at_bcp(Register reg, int bcp_offset) {

  assert(bcp_offset >= 0, "bcp is still pointing to start of bytecode");

  assert(reg != Rtemp, "should be different registers");

  ldrb(Rtemp, Address(Rbcp, bcp_offset));

  ldrb(reg, Address(Rbcp, bcp_offset+1));

  orr(reg, reg, AsmOperand(Rtemp, lsl, BitsPerByte));

}

void InterpreterMacroAssembler::get_index_at_bcp(Register index, int bcp_offset, Register tmp_reg, size_t index_size) {

  assert_different_registers(index, tmp_reg);

  if (index_size == sizeof(u2)) {

    // load bytes of index separately to avoid unaligned access

    ldrb(index, Address(Rbcp, bcp_offset+1));

    ldrb(tmp_reg, Address(Rbcp, bcp_offset));

    orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte));

  } else if (index_size == sizeof(u4)) {

    // TODO-AARCH64: consider using unaligned access here

    ldrb(index, Address(Rbcp, bcp_offset+3));

    ldrb(tmp_reg, Address(Rbcp, bcp_offset+2));

    orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte));

    ldrb(tmp_reg, Address(Rbcp, bcp_offset+1));

    orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte));

    ldrb(tmp_reg, Address(Rbcp, bcp_offset));

    orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte));

    // Check if the secondary index definition is still ~x, otherwise

    // we have to change the following assembler code to calculate the

    // plain index.

    assert(ConstantPool::decode_invokedynamic_index(~123) == 123, "else change next line");

    mvn_32(index, index);  // convert to plain index

  } else if (index_size == sizeof(u1)) {

    ldrb(index, Address(Rbcp, bcp_offset));

  } else {

    ShouldNotReachHere();

  }

}

// Sets cache, index.

void InterpreterMacroAssembler::get_cache_and_index_at_bcp(Register cache, Register index, int bcp_offset, size_t index_size) {

  assert(bcp_offset > 0, "bcp is still pointing to start of bytecode");

  assert_different_registers(cache, index);

  get_index_at_bcp(index, bcp_offset, cache, index_size);

  // load constant pool cache pointer

  ldr(cache, Address(FP, frame::interpreter_frame_cache_offset * wordSize));

  // convert from field index to ConstantPoolCacheEntry index

  assert(sizeof(ConstantPoolCacheEntry) == 4*wordSize, "adjust code below");

  // TODO-AARCH64 merge this shift with shift "add(..., Rcache, AsmOperand(Rindex, lsl, LogBytesPerWord))" after this method is called

  logical_shift_left(index, index, 2);

}

// Sets cache, index, bytecode.

void InterpreterMacroAssembler::get_cache_and_index_and_bytecode_at_bcp(Register cache, Register index, Register bytecode, int byte_no, int bcp_offset, size_t index_size) {

  get_cache_and_index_at_bcp(cache, index, bcp_offset, index_size);

  // caution index and bytecode can be the same

  add(bytecode, cache, AsmOperand(index, lsl, LogBytesPerWord));

#ifdef AARCH64

  add(bytecode, bytecode, (1 + byte_no) + in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::indices_offset()));

  ldarb(bytecode, bytecode);

#else

  ldrb(bytecode, Address(bytecode, (1 + byte_no) + in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::indices_offset())));

  TemplateTable::volatile_barrier(MacroAssembler::LoadLoad, noreg, true);

#endif // AARCH64

}

// Sets cache. Blows reg_tmp.

void InterpreterMacroAssembler::get_cache_entry_pointer_at_bcp(Register cache, Register reg_tmp, int bcp_offset, size_t index_size) {

  assert(bcp_offset > 0, "bcp is still pointing to start of bytecode");

  assert_different_registers(cache, reg_tmp);

  get_index_at_bcp(reg_tmp, bcp_offset, cache, index_size);

  // load constant pool cache pointer

  ldr(cache, Address(FP, frame::interpreter_frame_cache_offset * wordSize));

  // skip past the header

  add(cache, cache, in_bytes(ConstantPoolCache::base_offset()));

  // convert from field index to ConstantPoolCacheEntry index

  // and from word offset to byte offset

  assert(sizeof(ConstantPoolCacheEntry) == 4*wordSize, "adjust code below");

  add(cache, cache, AsmOperand(reg_tmp, lsl, 2 + LogBytesPerWord));

}

// Load object from cpool->resolved_references(index)

void InterpreterMacroAssembler::load_resolved_reference_at_index(

                                           Register result, Register index) {

  assert_different_registers(result, index);

  get_constant_pool(result);

  Register cache = result;

  // load pointer for resolved_references[] objArray

  ldr(cache, Address(result, ConstantPool::cache_offset_in_bytes()));

  ldr(cache, Address(result, ConstantPoolCache::resolved_references_offset_in_bytes()));

  resolve_oop_handle(cache);

  // Add in the index

  // convert from field index to resolved_references() index and from

  // word index to byte offset. Since this is a java object, it can be compressed

  add(cache, cache, AsmOperand(index, lsl, LogBytesPerHeapOop));

  load_heap_oop(result, Address(cache, arrayOopDesc::base_offset_in_bytes(T_OBJECT)));

}

void InterpreterMacroAssembler::load_resolved_klass_at_offset(

                                           Register Rcpool, Register Rindex, Register Rklass) {

  add(Rtemp, Rcpool, AsmOperand(Rindex, lsl, LogBytesPerWord));

  ldrh(Rtemp, Address(Rtemp, sizeof(ConstantPool))); // Rtemp = resolved_klass_index

  ldr(Rklass, Address(Rcpool,  ConstantPool::resolved_klasses_offset_in_bytes())); // Rklass = cpool->_resolved_klasses

  add(Rklass, Rklass, AsmOperand(Rtemp, lsl, LogBytesPerWord));

  ldr(Rklass, Address(Rklass, Array<Klass*>::base_offset_in_bytes()));

}

// Generate a subtype check: branch to not_subtype if sub_klass is

// not a subtype of super_klass.

// Profiling code for the subtype check failure (profile_typecheck_failed)

// should be explicitly generated by the caller in the not_subtype case.

// Blows Rtemp, tmp1, tmp2.

void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,

                                                  Register Rsuper_klass,

                                                  Label &not_subtype,

                                                  Register tmp1,

                                                  Register tmp2) {

  assert_different_registers(Rsub_klass, Rsuper_klass, tmp1, tmp2, Rtemp);

  Label ok_is_subtype, loop, update_cache;

  const Register super_check_offset = tmp1;

  const Register cached_super = tmp2;

  // Profile the not-null value's klass.

  profile_typecheck(tmp1, Rsub_klass);

  // Load the super-klass's check offset into

  ldr_u32(super_check_offset, Address(Rsuper_klass, Klass::super_check_offset_offset()));

  // Check for self

  cmp(Rsub_klass, Rsuper_klass);

  // Load from the sub-klass's super-class display list, or a 1-word cache of

  // the secondary superclass list, or a failing value with a sentinel offset

  // if the super-klass is an interface or exceptionally deep in the Java

  // hierarchy and we have to scan the secondary superclass list the hard way.

  // See if we get an immediate positive hit

  ldr(cached_super, Address(Rsub_klass, super_check_offset));

  cond_cmp(Rsuper_klass, cached_super, ne);

  b(ok_is_subtype, eq);

  // Check for immediate negative hit

  cmp(super_check_offset, in_bytes(Klass::secondary_super_cache_offset()));

  b(not_subtype, ne);

  // Now do a linear scan of the secondary super-klass chain.

  const Register supers_arr = tmp1;

  const Register supers_cnt = tmp2;

  const Register cur_super  = Rtemp;

  // Load objArrayOop of secondary supers.

  ldr(supers_arr, Address(Rsub_klass, Klass::secondary_supers_offset()));

  ldr_u32(supers_cnt, Address(supers_arr, Array<Klass*>::length_offset_in_bytes())); // Load the array length

#ifdef AARCH64

  cbz(supers_cnt, not_subtype);

  add(supers_arr, supers_arr, Array<Klass*>::base_offset_in_bytes());

#else

  cmp(supers_cnt, 0);

  // Skip to the start of array elements and prefetch the first super-klass.

  ldr(cur_super, Address(supers_arr, Array<Klass*>::base_offset_in_bytes(), pre_indexed), ne);

  b(not_subtype, eq);

#endif // AARCH64

  bind(loop);

#ifdef AARCH64

  ldr(cur_super, Address(supers_arr, wordSize, post_indexed));

#endif // AARCH64

  cmp(cur_super, Rsuper_klass);

  b(update_cache, eq);

  subs(supers_cnt, supers_cnt, 1);

#ifndef AARCH64

  ldr(cur_super, Address(supers_arr, wordSize, pre_indexed), ne);

#endif // !AARCH64

  b(loop, ne);

  b(not_subtype);

  bind(update_cache);

  // Must be equal but missed in cache.  Update cache.

  str(Rsuper_klass, Address(Rsub_klass, Klass::secondary_super_cache_offset()));

  bind(ok_is_subtype);

}

// The 1st part of the store check.

// Sets card_table_base register.

void InterpreterMacroAssembler::store_check_part1(Register card_table_base) {

  // Check barrier set type (should be card table) and element size

  BarrierSet* bs = Universe::heap()->barrier_set();

         "Wrong barrier set kind");

  CardTable* ct = ctbs->card_table();

  assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "Adjust store check code");

  // Load card table base address.

  /* Performance note.

     There is an alternative way of loading card table base address

     from thread descriptor, which may look more efficient:

     ldr(card_table_base, Address(Rthread, JavaThread::card_table_base_offset()));

     However, performance measurements of micro benchmarks and specJVM98

     showed that loading of card table base from thread descriptor is

     7-18% slower compared to loading of literal embedded into the code.

     Possible cause is a cache miss (card table base address resides in a

     rarely accessed area of thread descriptor).

  */

  // TODO-AARCH64 Investigate if mov_slow is faster than ldr from Rthread on AArch64

  mov_address(card_table_base, (address)ct->byte_map_base(), symbolic_Relocation::card_table_reference);

}

// The 2nd part of the store check.

void InterpreterMacroAssembler::store_check_part2(Register obj, Register card_table_base, Register tmp) {

  assert_different_registers(obj, card_table_base, tmp);

  assert(CardTable::dirty_card_val() == 0, "Dirty card value must be 0 due to optimizations.");

#ifdef AARCH64

  add(card_table_base, card_table_base, AsmOperand(obj, lsr, CardTable::card_shift));

  Address card_table_addr(card_table_base);

#else

  Address card_table_addr(card_table_base, obj, lsr, CardTable::card_shift);

#endif

  if (UseCondCardMark) {

    if (UseConcMarkSweepGC) {

      membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad), noreg);

    }

    Label already_dirty;

    ldrb(tmp, card_table_addr);

    cbz(tmp, already_dirty);

    set_card(card_table_base, card_table_addr, tmp);

    bind(already_dirty);

  } else {

    if (UseConcMarkSweepGC && CMSPrecleaningEnabled) {

      membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore), noreg);

    }

    set_card(card_table_base, card_table_addr, tmp);

  }

}

void InterpreterMacroAssembler::set_card(Register card_table_base, Address card_table_addr, Register tmp) {

#ifdef AARCH64

  strb(ZR, card_table_addr);

#else

  CardTable* ct = ctbs->card_table();

  if ((((uintptr_t)ct->byte_map_base() & 0xff) == 0)) {

    // Card table is aligned so the lowest byte of the table address base is zero.

    // This works only if the code is not saved for later use, possibly

    // in a context where the base would no longer be aligned.

    strb(card_table_base, card_table_addr);

  } else {

    mov(tmp, 0);

    strb(tmp, card_table_addr);

  }

#endif // AARCH64

}

//////////////////////////////////////////////////////////////////////////////////

// Java Expression Stack

void InterpreterMacroAssembler::pop_ptr(Register r) {

  assert(r != Rstack_top, "unpredictable instruction");

  ldr(r, Address(Rstack_top, wordSize, post_indexed));

}

void InterpreterMacroAssembler::pop_i(Register r) {

  assert(r != Rstack_top, "unpredictable instruction");

  ldr_s32(r, Address(Rstack_top, wordSize, post_indexed));

  zap_high_non_significant_bits(r);

}

#ifdef AARCH64

void InterpreterMacroAssembler::pop_l(Register r) {

  assert(r != Rstack_top, "unpredictable instruction");

  ldr(r, Address(Rstack_top, 2*wordSize, post_indexed));

}

#else

void InterpreterMacroAssembler::pop_l(Register lo, Register hi) {

  assert_different_registers(lo, hi);

  assert(lo < hi, "lo must be < hi");

  pop(RegisterSet(lo) | RegisterSet(hi));

}

#endif // AARCH64

void InterpreterMacroAssembler::pop_f(FloatRegister fd) {

#ifdef AARCH64

  ldr_s(fd, Address(Rstack_top, wordSize, post_indexed));

#else

  fpops(fd);

#endif // AARCH64

}