/*

 * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "asm/assembler.hpp"

#include "asm/assembler.inline.hpp"

#include "gc/shared/cardTableModRefBS.hpp"

#include "gc/shared/collectedHeap.inline.hpp"

#include "interpreter/interpreter.hpp"

#include "memory/resourceArea.hpp"

#include "prims/methodHandles.hpp"

#include "runtime/biasedLocking.hpp"

#include "runtime/interfaceSupport.hpp"

#include "runtime/objectMonitor.hpp"

#include "runtime/os.hpp"

#include "runtime/sharedRuntime.hpp"

#include "runtime/stubRoutines.hpp"

#include "utilities/macros.hpp"

#if INCLUDE_ALL_GCS

#include "gc/g1/g1CollectedHeap.inline.hpp"

#include "gc/g1/g1SATBCardTableModRefBS.hpp"

#include "gc/g1/heapRegion.hpp"

#endif // INCLUDE_ALL_GCS

#ifdef PRODUCT

#define BLOCK_COMMENT(str) /* nothing */

#define STOP(error) stop(error)

#else

#define BLOCK_COMMENT(str) block_comment(str)

#define STOP(error) block_comment(error); stop(error)

#endif

#define BIND(label) bind(label); BLOCK_COMMENT(#label ":")

// Implementation of AddressLiteral

// A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.

unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {

  // -----------------Table 4.5 -------------------- //

  16, 32, 64,  // EVEX_FV(0)

  4,  4,  4,   // EVEX_FV(1) - with Evex.b

  16, 32, 64,  // EVEX_FV(2) - with Evex.w

  8,  8,  8,   // EVEX_FV(3) - with Evex.w and Evex.b

  8,  16, 32,  // EVEX_HV(0)

  4,  4,  4,   // EVEX_HV(1) - with Evex.b

  // -----------------Table 4.6 -------------------- //

  16, 32, 64,  // EVEX_FVM(0)

  1,  1,  1,   // EVEX_T1S(0)

  2,  2,  2,   // EVEX_T1S(1)

  4,  4,  4,   // EVEX_T1S(2)

  8,  8,  8,   // EVEX_T1S(3)

  4,  4,  4,   // EVEX_T1F(0)

  8,  8,  8,   // EVEX_T1F(1)

  8,  8,  8,   // EVEX_T2(0)

  0,  16, 16,  // EVEX_T2(1)

  0,  16, 16,  // EVEX_T4(0)

  0,  0,  32,  // EVEX_T4(1)

  0,  0,  32,  // EVEX_T8(0)

  8,  16, 32,  // EVEX_HVM(0)

  4,  8,  16,  // EVEX_QVM(0)

  2,  4,  8,   // EVEX_OVM(0)

  16, 16, 16,  // EVEX_M128(0)

  8,  32, 64,  // EVEX_DUP(0)

  0,  0,  0    // EVEX_NTUP

};

AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {

  _is_lval = false;

  _target = target;

  switch (rtype) {

  case relocInfo::oop_type:

  case relocInfo::metadata_type:

    // Oops are a special case. Normally they would be their own section

    // but in cases like icBuffer they are literals in the code stream that

    // we don't have a section for. We use none so that we get a literal address

    // which is always patchable.

    break;

  case relocInfo::external_word_type:

    _rspec = external_word_Relocation::spec(target);

    break;

  case relocInfo::internal_word_type:

    _rspec = internal_word_Relocation::spec(target);

    break;

  case relocInfo::opt_virtual_call_type:

    _rspec = opt_virtual_call_Relocation::spec();

    break;

  case relocInfo::static_call_type:

    _rspec = static_call_Relocation::spec();

    break;

  case relocInfo::runtime_call_type:

    _rspec = runtime_call_Relocation::spec();

    break;

  case relocInfo::poll_type:

  case relocInfo::poll_return_type:

    _rspec = Relocation::spec_simple(rtype);

    break;

  case relocInfo::none:

    break;

  default:

    ShouldNotReachHere();

    break;

  }

}

// Implementation of Address

#ifdef _LP64

Address Address::make_array(ArrayAddress adr) {

  // Not implementable on 64bit machines

  // Should have been handled higher up the call chain.

  ShouldNotReachHere();

  return Address();

}

// exceedingly dangerous constructor

Address::Address(int disp, address loc, relocInfo::relocType rtype) {

  _base  = noreg;

  _index = noreg;

  _scale = no_scale;

  _disp  = disp;

  switch (rtype) {

    case relocInfo::external_word_type:

      _rspec = external_word_Relocation::spec(loc);

      break;

    case relocInfo::internal_word_type:

      _rspec = internal_word_Relocation::spec(loc);

      break;

    case relocInfo::runtime_call_type:

      // HMM

      _rspec = runtime_call_Relocation::spec();

      break;

    case relocInfo::poll_type:

    case relocInfo::poll_return_type:

      _rspec = Relocation::spec_simple(rtype);

      break;

    case relocInfo::none:

      break;

    default:

      ShouldNotReachHere();

  }

}

#else // LP64

Address Address::make_array(ArrayAddress adr) {

  AddressLiteral base = adr.base();

  Address index = adr.index();

  assert(index._disp == 0, "must not have disp"); // maybe it can?

  Address array(index._base, index._index, index._scale, (intptr_t) base.target());

  array._rspec = base._rspec;

  return array;

}

// exceedingly dangerous constructor

Address::Address(address loc, RelocationHolder spec) {

  _base  = noreg;

  _index = noreg;

  _scale = no_scale;

  _disp  = (intptr_t) loc;

  _rspec = spec;

}

#endif // _LP64

// Convert the raw encoding form into the form expected by the constructor for

// Address.  An index of 4 (rsp) corresponds to having no index, so convert

// that to noreg for the Address constructor.

Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {

  RelocationHolder rspec;

  if (disp_reloc != relocInfo::none) {

    rspec = Relocation::spec_simple(disp_reloc);

  }

  bool valid_index = index != rsp->encoding();

  if (valid_index) {

    Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));

    madr._rspec = rspec;

    return madr;

  } else {

    Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));

    madr._rspec = rspec;

    return madr;

  }

}

// Implementation of Assembler

int AbstractAssembler::code_fill_byte() {

  return (u_char)'\xF4'; // hlt

}

// make this go away someday

void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {

  if (rtype == relocInfo::none)

    emit_int32(data);

  else

    emit_data(data, Relocation::spec_simple(rtype), format);

}

void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {

  assert(imm_operand == 0, "default format must be immediate in this file");

  assert(inst_mark() != NULL, "must be inside InstructionMark");

  if (rspec.type() !=  relocInfo::none) {

    #ifdef ASSERT

      check_relocation(rspec, format);

    #endif

    // Do not use AbstractAssembler::relocate, which is not intended for

    // embedded words.  Instead, relocate to the enclosing instruction.

    // hack. call32 is too wide for mask so use disp32

    if (format == call32_operand)

      code_section()->relocate(inst_mark(), rspec, disp32_operand);

    else

      code_section()->relocate(inst_mark(), rspec, format);

  }

  emit_int32(data);

}

static int encode(Register r) {

  int enc = r->encoding();

  if (enc >= 8) {

    enc -= 8;

  }

  return enc;

}

void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {

  assert(dst->has_byte_register(), "must have byte register");

  assert(isByte(op1) && isByte(op2), "wrong opcode");

  assert(isByte(imm8), "not a byte");

  assert((op1 & 0x01) == 0, "should be 8bit operation");

  emit_int8(op1);

  emit_int8(op2 | encode(dst));

  emit_int8(imm8);

}

void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {

  assert(isByte(op1) && isByte(op2), "wrong opcode");

  assert((op1 & 0x01) == 1, "should be 32bit operation");

  assert((op1 & 0x02) == 0, "sign-extension bit should not be set");

  if (is8bit(imm32)) {

    emit_int8(op1 | 0x02); // set sign bit

    emit_int8(op2 | encode(dst));

    emit_int8(imm32 & 0xFF);

  } else {

    emit_int8(op1);

    emit_int8(op2 | encode(dst));

    emit_int32(imm32);

  }

}

// Force generation of a 4 byte immediate value even if it fits into 8bit

void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {

  assert(isByte(op1) && isByte(op2), "wrong opcode");

  assert((op1 & 0x01) == 1, "should be 32bit operation");

  assert((op1 & 0x02) == 0, "sign-extension bit should not be set");

  emit_int8(op1);

  emit_int8(op2 | encode(dst));

  emit_int32(imm32);

}

// immediate-to-memory forms

void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {

  assert((op1 & 0x01) == 1, "should be 32bit operation");

  assert((op1 & 0x02) == 0, "sign-extension bit should not be set");

  if (is8bit(imm32)) {

    emit_int8(op1 | 0x02); // set sign bit

    emit_operand(rm, adr, 1);

    emit_int8(imm32 & 0xFF);

  } else {

    emit_int8(op1);

    emit_operand(rm, adr, 4);

    emit_int32(imm32);

  }

}

void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {

  assert(isByte(op1) && isByte(op2), "wrong opcode");

  emit_int8(op1);

  emit_int8(op2 | encode(dst) << 3 | encode(src));

}

bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,

                                           int cur_tuple_type, int in_size_in_bits, int cur_encoding) {

  int mod_idx = 0;

  // We will test if the displacement fits the compressed format and if so

  // apply the compression to the displacment iff the result is8bit.

  if (VM_Version::supports_evex() && is_evex_inst) {

    switch (cur_tuple_type) {

    case EVEX_FV:

      if ((cur_encoding & VEX_W) == VEX_W) {

        mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;

      } else {

        mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;

      }

      break;

    case EVEX_HV:

      mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;

      break;

    case EVEX_FVM:

      break;

    case EVEX_T1S:

      switch (in_size_in_bits) {

      case EVEX_8bit:

        break;

      case EVEX_16bit:

        mod_idx = 1;

        break;

      case EVEX_32bit:

        mod_idx = 2;

        break;

      case EVEX_64bit:

        mod_idx = 3;

        break;

      }

      break;

    case EVEX_T1F:

    case EVEX_T2:

    case EVEX_T4:

      mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;

      break;

    case EVEX_T8:

      break;

    case EVEX_HVM:

      break;

    case EVEX_QVM:

      break;

    case EVEX_OVM:

      break;

    case EVEX_M128:

      break;

    case EVEX_DUP:

      break;

    default:

      assert(0, "no valid evex tuple_table entry");

      break;

    }

    if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {

      int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];

      if ((disp % disp_factor) == 0) {

        int new_disp = disp / disp_factor;

        if ((-0x80 <= new_disp && new_disp < 0x80)) {

          disp = new_disp;

        }

      } else {

        return false;

      }

    }

  }

  return (-0x80 <= disp && disp < 0x80);

}

bool Assembler::emit_compressed_disp_byte(int &disp) {

  int mod_idx = 0;

  // We will test if the displacement fits the compressed format and if so

  // apply the compression to the displacment iff the result is8bit.

  if (VM_Version::supports_evex() && (_attributes != NULL) && _attributes->is_evex_instruction()) {

    int evex_encoding = _attributes->get_evex_encoding();

    int tuple_type = _attributes->get_tuple_type();

    switch (tuple_type) {

    case EVEX_FV:

      if ((evex_encoding & VEX_W) == VEX_W) {

        mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;

      } else {

        mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;

      }

      break;

    case EVEX_HV:

      mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;

      break;

    case EVEX_FVM:

      break;

    case EVEX_T1S:

      switch (_attributes->get_input_size()) {

      case EVEX_8bit:

        break;

      case EVEX_16bit:

        mod_idx = 1;

        break;

      case EVEX_32bit:

        mod_idx = 2;

        break;

      case EVEX_64bit:

        mod_idx = 3;

        break;

      }

      break;

    case EVEX_T1F:

    case EVEX_T2:

    case EVEX_T4:

      mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;

      break;

    case EVEX_T8:

      break;

    case EVEX_HVM:

      break;

    case EVEX_QVM:

      break;

    case EVEX_OVM:

      break;

    case EVEX_M128:

      break;

    case EVEX_DUP:

      break;

    default:

      assert(0, "no valid evex tuple_table entry");

      break;

    }

    int vector_len = _attributes->get_vector_len();

    if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {

      int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];

      if ((disp % disp_factor) == 0) {

        int new_disp = disp / disp_factor;

        if (is8bit(new_disp)) {

          disp = new_disp;

        }

      } else {

        return false;

      }

    }

  }

  return is8bit(disp);

}

void Assembler::emit_operand(Register reg, Register base, Register index,

                             Address::ScaleFactor scale, int disp,

                             RelocationHolder const& rspec,

                             int rip_relative_correction) {

  relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();

  // Encode the registers as needed in the fields they are used in

  int regenc = encode(reg) << 3;

  int indexenc = index->is_valid() ? encode(index) << 3 : 0;

  int baseenc = base->is_valid() ? encode(base) : 0;

  if (base->is_valid()) {

    if (index->is_valid()) {

      assert(scale != Address::no_scale, "inconsistent address");

      // [base + index*scale + disp]

      if (disp == 0 && rtype == relocInfo::none  &&

          base != rbp LP64_ONLY(&& base != r13)) {

        // [base + index*scale]