/*

 * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "asm/macroAssembler.hpp"

#include "asm/macroAssembler.inline.hpp"

#include "c1/c1_Compilation.hpp"

#include "c1/c1_LIRAssembler.hpp"

#include "c1/c1_MacroAssembler.hpp"

#include "c1/c1_Runtime1.hpp"

#include "c1/c1_ValueStack.hpp"

#include "ci/ciArrayKlass.hpp"

#include "ci/ciInstance.hpp"

#include "gc/shared/barrierSet.hpp"

#include "gc/shared/cardTableBarrierSet.hpp"

#include "gc/shared/collectedHeap.hpp"

#include "nativeInst_x86.hpp"

#include "oops/objArrayKlass.hpp"

#include "runtime/frame.inline.hpp"

#include "runtime/safepointMechanism.hpp"

#include "runtime/sharedRuntime.hpp"

#include "vmreg_x86.inline.hpp"

// These masks are used to provide 128-bit aligned bitmasks to the XMM

// instructions, to allow sign-masking or sign-bit flipping.  They allow

// fast versions of NegF/NegD and AbsF/AbsD.

// Note: 'double' and 'long long' have 32-bits alignment on x86.

static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {

  // Use the expression (adr)&(~0xF) to provide 128-bits aligned address

  // of 128-bits operands for SSE instructions.

  jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));

  // Store the value to a 128-bits operand.

  operand[0] = lo;

  operand[1] = hi;

  return operand;

}

// Buffer for 128-bits masks used by SSE instructions.

static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)

// Static initialization during VM startup.

static jlong *float_signmask_pool  = double_quadword(&fp_signmask_pool[1*2],         CONST64(0x7FFFFFFF7FFFFFFF),         CONST64(0x7FFFFFFF7FFFFFFF));

static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2],         CONST64(0x7FFFFFFFFFFFFFFF),         CONST64(0x7FFFFFFFFFFFFFFF));

static jlong *float_signflip_pool  = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));

static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));

NEEDS_CLEANUP // remove this definitions ?

const Register IC_Klass    = rax;   // where the IC klass is cached

const Register SYNC_header = rax;   // synchronization header

const Register SHIFT_count = rcx;   // where count for shift operations must be

#define __ _masm->

static void select_different_registers(Register preserve,

                                       Register extra,

                                       Register &tmp1,

                                       Register &tmp2) {

  if (tmp1 == preserve) {

    assert_different_registers(tmp1, tmp2, extra);

    tmp1 = extra;

  } else if (tmp2 == preserve) {

    assert_different_registers(tmp1, tmp2, extra);

    tmp2 = extra;

  }

  assert_different_registers(preserve, tmp1, tmp2);

}

static void select_different_registers(Register preserve,

                                       Register extra,

                                       Register &tmp1,

                                       Register &tmp2,

                                       Register &tmp3) {

  if (tmp1 == preserve) {

    assert_different_registers(tmp1, tmp2, tmp3, extra);

    tmp1 = extra;

  } else if (tmp2 == preserve) {

    assert_different_registers(tmp1, tmp2, tmp3, extra);

    tmp2 = extra;

  } else if (tmp3 == preserve) {

    assert_different_registers(tmp1, tmp2, tmp3, extra);

    tmp3 = extra;

  }

  assert_different_registers(preserve, tmp1, tmp2, tmp3);

}

bool LIR_Assembler::is_small_constant(LIR_Opr opr) {

  if (opr->is_constant()) {

    LIR_Const* constant = opr->as_constant_ptr();

    switch (constant->type()) {

      case T_INT: {

        return true;

      }

      default:

        return false;

    }

  }

  return false;

}

LIR_Opr LIR_Assembler::receiverOpr() {

  return FrameMap::receiver_opr;

}

LIR_Opr LIR_Assembler::osrBufferPointer() {

  return FrameMap::as_pointer_opr(receiverOpr()->as_register());

}

//--------------fpu register translations-----------------------

address LIR_Assembler::float_constant(float f) {

  address const_addr = __ float_constant(f);

  if (const_addr == NULL) {

    bailout("const section overflow");

    return __ code()->consts()->start();

  } else {

    return const_addr;

  }

}

address LIR_Assembler::double_constant(double d) {

  address const_addr = __ double_constant(d);

  if (const_addr == NULL) {

    bailout("const section overflow");

    return __ code()->consts()->start();

  } else {

    return const_addr;

  }

}

void LIR_Assembler::set_24bit_FPU() {

  __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24()));

}

void LIR_Assembler::reset_FPU() {

  __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));

}

void LIR_Assembler::fpop() {

  __ fpop();

}

void LIR_Assembler::fxch(int i) {

  __ fxch(i);

}

void LIR_Assembler::fld(int i) {

  __ fld_s(i);

}

void LIR_Assembler::ffree(int i) {

  __ ffree(i);

}

void LIR_Assembler::breakpoint() {

  __ int3();

}

void LIR_Assembler::push(LIR_Opr opr) {

  if (opr->is_single_cpu()) {

    __ push_reg(opr->as_register());

  } else if (opr->is_double_cpu()) {

    NOT_LP64(__ push_reg(opr->as_register_hi()));

    __ push_reg(opr->as_register_lo());

  } else if (opr->is_stack()) {

    __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));

  } else if (opr->is_constant()) {

    LIR_Const* const_opr = opr->as_constant_ptr();

    if (const_opr->type() == T_OBJECT) {

      __ push_oop(const_opr->as_jobject());

    } else if (const_opr->type() == T_INT) {

      __ push_jint(const_opr->as_jint());

    } else {

      ShouldNotReachHere();

    }

  } else {

    ShouldNotReachHere();

  }

}

void LIR_Assembler::pop(LIR_Opr opr) {

  if (opr->is_single_cpu()) {

    __ pop_reg(opr->as_register());

  } else {

    ShouldNotReachHere();

  }

}

bool LIR_Assembler::is_literal_address(LIR_Address* addr) {

  return addr->base()->is_illegal() && addr->index()->is_illegal();

}

//-------------------------------------------

Address LIR_Assembler::as_Address(LIR_Address* addr) {

  return as_Address(addr, rscratch1);

}

Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {

  if (addr->base()->is_illegal()) {

    assert(addr->index()->is_illegal(), "must be illegal too");

    AddressLiteral laddr((address)addr->disp(), relocInfo::none);

    if (! __ reachable(laddr)) {

      __ movptr(tmp, laddr.addr());

      Address res(tmp, 0);

      return res;

    } else {

      return __ as_Address(laddr);

    }

  }

  Register base = addr->base()->as_pointer_register();

  if (addr->index()->is_illegal()) {

    return Address( base, addr->disp());

  } else if (addr->index()->is_cpu_register()) {

    Register index = addr->index()->as_pointer_register();

    return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp());

  } else if (addr->index()->is_constant()) {

    intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();

    assert(Assembler::is_simm32(addr_offset), "must be");

    return Address(base, addr_offset);

  } else {

    Unimplemented();

    return Address();

  }

}

Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {

  Address base = as_Address(addr);

  return Address(base._base, base._index, base._scale, base._disp + BytesPerWord);

}

Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {

  return as_Address(addr);

}

void LIR_Assembler::osr_entry() {

  offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());

  BlockBegin* osr_entry = compilation()->hir()->osr_entry();

  ValueStack* entry_state = osr_entry->state();

  int number_of_locks = entry_state->locks_size();

  // we jump here if osr happens with the interpreter

  // state set up to continue at the beginning of the

  // loop that triggered osr - in particular, we have

  // the following registers setup:

  //

  // rcx: osr buffer

  //

  // build frame

  ciMethod* m = compilation()->method();

  __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());

  // OSR buffer is

  //

  // locals[nlocals-1..0]

  // monitors[0..number_of_locks]

  //

  // locals is a direct copy of the interpreter frame so in the osr buffer

  // so first slot in the local array is the last local from the interpreter

  // and last slot is local[0] (receiver) from the interpreter

  //

  // Similarly with locks. The first lock slot in the osr buffer is the nth lock

  // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock

  // in the interpreter frame (the method lock if a sync method)

  // Initialize monitors in the compiled activation.

  //   rcx: pointer to osr buffer

  //

  // All other registers are dead at this point and the locals will be

  // copied into place by code emitted in the IR.

  Register OSR_buf = osrBufferPointer()->as_pointer_register();

  { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");

    int monitor_offset = BytesPerWord * method()->max_locals() +

      (BasicObjectLock::size() * BytesPerWord) * (number_of_locks - 1);

    // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in

    // the OSR buffer using 2 word entries: first the lock and then

    // the oop.

    for (int i = 0; i < number_of_locks; i++) {

      int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);

#ifdef ASSERT

      // verify the interpreter's monitor has a non-null object

      {

        Label L;

        __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), (int32_t)NULL_WORD);

        __ jcc(Assembler::notZero, L);

        __ stop("locked object is NULL");

        __ bind(L);

      }

#endif

      __ movptr(rbx, Address(OSR_buf, slot_offset + 0));

      __ movptr(frame_map()->address_for_monitor_lock(i), rbx);

      __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));

      __ movptr(frame_map()->address_for_monitor_object(i), rbx);

    }

  }

}

// inline cache check; done before the frame is built.

int LIR_Assembler::check_icache() {

  Register receiver = FrameMap::receiver_opr->as_register();

  Register ic_klass = IC_Klass;

  const int ic_cmp_size = LP64_ONLY(10) NOT_LP64(9);

  const bool do_post_padding = VerifyOops || UseCompressedClassPointers;

  if (!do_post_padding) {

    // insert some nops so that the verified entry point is aligned on CodeEntryAlignment

    __ align(CodeEntryAlignment, __ offset() + ic_cmp_size);

  }

  int offset = __ offset();

  __ inline_cache_check(receiver, IC_Klass);

  assert(__ offset() % CodeEntryAlignment == 0 || do_post_padding, "alignment must be correct");

  if (do_post_padding) {

    // force alignment after the cache check.

    // It's been verified to be aligned if !VerifyOops

    __ align(CodeEntryAlignment);

  }

  return offset;

}

void LIR_Assembler::clinit_barrier(ciMethod* method) {

  assert(VM_Version::supports_fast_class_init_checks(), "sanity");

  assert(!method->holder()->is_not_initialized(), "initialization should have been started");

  Label L_skip_barrier;

  Register klass = rscratch1;

  Register thread = LP64_ONLY( r15_thread ) NOT_LP64( noreg );

  assert(thread != noreg, "x86_32 not implemented");

  __ mov_metadata(klass, method->holder()->constant_encoding());

  __ clinit_barrier(klass, thread, &L_skip_barrier /*L_fast_path*/);

  __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));

  __ bind(L_skip_barrier);

}

void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {

  jobject o = NULL;

  PatchingStub* patch = new PatchingStub(_masm, patching_id(info));

  __ movoop(reg, o);

  patching_epilog(patch, lir_patch_normal, reg, info);

}

void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {

  Metadata* o = NULL;

  PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);

  __ mov_metadata(reg, o);

  patching_epilog(patch, lir_patch_normal, reg, info);

}

// This specifies the rsp decrement needed to build the frame

int LIR_Assembler::initial_frame_size_in_bytes() const {

  // if rounding, must let FrameMap know!

  // The frame_map records size in slots (32bit word)

  // subtract two words to account for return address and link

  return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word))  * VMRegImpl::stack_slot_size;

}

int LIR_Assembler::emit_exception_handler() {

  // if the last instruction is a call (typically to do a throw which

  // is coming at the end after block reordering) the return address

  // must still point into the code area in order to avoid assertion

  // failures when searching for the corresponding bci => add a nop

  // (was bug 5/14/1999 - gri)

  __ nop();

  // generate code for exception handler

  address handler_base = __ start_a_stub(exception_handler_size());

  if (handler_base == NULL) {

    // not enough space left for the handler

    bailout("exception handler overflow");

    return -1;

  }

  int offset = code_offset();

  // the exception oop and pc are in rax, and rdx

  // no other registers need to be preserved, so invalidate them

  __ invalidate_registers(false, true, true, false, true, true);

  // check that there is really an exception

  __ verify_not_null_oop(rax);

  // search an exception handler (rax: exception oop, rdx: throwing pc)

  __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));

  __ should_not_reach_here();

  guarantee(code_offset() - offset <= exception_handler_size(), "overflow");

  __ end_a_stub();

  return offset;

}

// Emit the code to remove the frame from the stack in the exception

// unwind path.

int LIR_Assembler::emit_unwind_handler() {

#ifndef PRODUCT

  if (CommentedAssembly) {

    _masm->block_comment("Unwind handler");

  }

#endif

  int offset = code_offset();

  // Fetch the exception from TLS and clear out exception related thread state

  Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread);

  NOT_LP64(__ get_thread(rsi));

  __ movptr(rax, Address(thread, JavaThread::exception_oop_offset()));

  __ movptr(Address(thread, JavaThread::exception_oop_offset()), (intptr_t)NULL_WORD);

  __ movptr(Address(thread, JavaThread::exception_pc_offset()), (intptr_t)NULL_WORD);

  __ bind(_unwind_handler_entry);

  __ verify_not_null_oop(rax);

  if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {

    __ mov(rbx, rax);  // Preserve the exception (rbx is always callee-saved)

  }

  // Preform needed unlocking

  MonitorExitStub* stub = NULL;

  if (method()->is_synchronized()) {

    monitor_address(0, FrameMap::rax_opr);

    stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);

    __ unlock_object(rdi, rsi, rax, *stub->entry());

    __ bind(*stub->continuation());

  }

  if (compilation()->env()->dtrace_method_probes()) {

#ifdef _LP64

    __ mov(rdi, r15_thread);

    __ mov_metadata(rsi, method()->constant_encoding());

#else

    __ get_thread(rax);

    __ movptr(Address(rsp, 0), rax);

    __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding());

#endif

    __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));

  }

  if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {

    __ mov(rax, rbx);  // Restore the exception

  }

  // remove the activation and dispatch to the unwind handler

  __ remove_frame(initial_frame_size_in_bytes());

  __ jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));

  // Emit the slow path assembly

  if (stub != NULL) {

    stub->emit_code(this);

  }

  return offset;

}

int LIR_Assembler::emit_deopt_handler() {

  // if the last instruction is a call (typically to do a throw which

  // is coming at the end after block reordering) the return address

  // must still point into the code area in order to avoid assertion

  // failures when searching for the corresponding bci => add a nop

  // (was bug 5/14/1999 - gri)

  __ nop();

  // generate code for exception handler