/*

 * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "assembler_x86.inline.hpp"

#include "gc_interface/collectedHeap.inline.hpp"

#include "interpreter/interpreter.hpp"

#include "memory/cardTableModRefBS.hpp"

#include "memory/resourceArea.hpp"

#include "prims/methodHandles.hpp"

#include "runtime/biasedLocking.hpp"

#include "runtime/interfaceSupport.hpp"

#include "runtime/objectMonitor.hpp"

#include "runtime/os.hpp"

#include "runtime/sharedRuntime.hpp"

#include "runtime/stubRoutines.hpp"

#ifndef SERIALGC

#include "gc_implementation/g1/g1CollectedHeap.inline.hpp"

#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp"

#include "gc_implementation/g1/heapRegion.hpp"

#endif

// Implementation of AddressLiteral

AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {

  _is_lval = false;

  _target = target;

  switch (rtype) {

  case relocInfo::oop_type:

    // Oops are a special case. Normally they would be their own section

    // but in cases like icBuffer they are literals in the code stream that

    // we don't have a section for. We use none so that we get a literal address

    // which is always patchable.

    break;

  case relocInfo::external_word_type:

    _rspec = external_word_Relocation::spec(target);

    break;

  case relocInfo::internal_word_type:

    _rspec = internal_word_Relocation::spec(target);

    break;

  case relocInfo::opt_virtual_call_type:

    _rspec = opt_virtual_call_Relocation::spec();

    break;

  case relocInfo::static_call_type:

    _rspec = static_call_Relocation::spec();

    break;

  case relocInfo::runtime_call_type:

    _rspec = runtime_call_Relocation::spec();

    break;

  case relocInfo::poll_type:

  case relocInfo::poll_return_type:

    _rspec = Relocation::spec_simple(rtype);

    break;

  case relocInfo::none:

    break;

  default:

    ShouldNotReachHere();

    break;

  }

}

// Implementation of Address

#ifdef _LP64

Address Address::make_array(ArrayAddress adr) {

  // Not implementable on 64bit machines

  // Should have been handled higher up the call chain.

  ShouldNotReachHere();

  return Address();

}

// exceedingly dangerous constructor

Address::Address(int disp, address loc, relocInfo::relocType rtype) {

  _base  = noreg;

  _index = noreg;

  _scale = no_scale;

  _disp  = disp;

  switch (rtype) {

    case relocInfo::external_word_type:

      _rspec = external_word_Relocation::spec(loc);

      break;

    case relocInfo::internal_word_type:

      _rspec = internal_word_Relocation::spec(loc);

      break;

    case relocInfo::runtime_call_type:

      // HMM

      _rspec = runtime_call_Relocation::spec();

      break;

    case relocInfo::poll_type:

    case relocInfo::poll_return_type:

      _rspec = Relocation::spec_simple(rtype);

      break;

    case relocInfo::none:

      break;

    default:

      ShouldNotReachHere();

  }

}

#else // LP64

Address Address::make_array(ArrayAddress adr) {

  AddressLiteral base = adr.base();

  Address index = adr.index();

  assert(index._disp == 0, "must not have disp"); // maybe it can?

  Address array(index._base, index._index, index._scale, (intptr_t) base.target());

  array._rspec = base._rspec;

  return array;

}

// exceedingly dangerous constructor

Address::Address(address loc, RelocationHolder spec) {

  _base  = noreg;

  _index = noreg;

  _scale = no_scale;

  _disp  = (intptr_t) loc;

  _rspec = spec;

}

#endif // _LP64

// Convert the raw encoding form into the form expected by the constructor for

// Address.  An index of 4 (rsp) corresponds to having no index, so convert

// that to noreg for the Address constructor.

Address Address::make_raw(int base, int index, int scale, int disp, bool disp_is_oop) {

  RelocationHolder rspec;

  if (disp_is_oop) {

    rspec = Relocation::spec_simple(relocInfo::oop_type);

  }

  bool valid_index = index != rsp->encoding();

  if (valid_index) {

    Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));

    madr._rspec = rspec;

    return madr;

  } else {

    Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));

    madr._rspec = rspec;

    return madr;

  }

}

// Implementation of Assembler

int AbstractAssembler::code_fill_byte() {

  return (u_char)'\xF4'; // hlt

}

// make this go away someday

void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {

  if (rtype == relocInfo::none)

        emit_long(data);

  else  emit_data(data, Relocation::spec_simple(rtype), format);

}

void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {

  assert(imm_operand == 0, "default format must be immediate in this file");

  assert(inst_mark() != NULL, "must be inside InstructionMark");

  if (rspec.type() !=  relocInfo::none) {

    #ifdef ASSERT

      check_relocation(rspec, format);

    #endif

    // Do not use AbstractAssembler::relocate, which is not intended for

    // embedded words.  Instead, relocate to the enclosing instruction.

    // hack. call32 is too wide for mask so use disp32

    if (format == call32_operand)

      code_section()->relocate(inst_mark(), rspec, disp32_operand);

    else

      code_section()->relocate(inst_mark(), rspec, format);

  }

  emit_long(data);

}

static int encode(Register r) {

  int enc = r->encoding();

  if (enc >= 8) {

    enc -= 8;

  }

  return enc;

}

static int encode(XMMRegister r) {

  int enc = r->encoding();

  if (enc >= 8) {

    enc -= 8;

  }

  return enc;

}

void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {

  assert(dst->has_byte_register(), "must have byte register");

  assert(isByte(op1) && isByte(op2), "wrong opcode");

  assert(isByte(imm8), "not a byte");

  assert((op1 & 0x01) == 0, "should be 8bit operation");

  emit_byte(op1);

  emit_byte(op2 | encode(dst));

  emit_byte(imm8);

}

void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {

  assert(isByte(op1) && isByte(op2), "wrong opcode");

  assert((op1 & 0x01) == 1, "should be 32bit operation");

  assert((op1 & 0x02) == 0, "sign-extension bit should not be set");

  if (is8bit(imm32)) {

    emit_byte(op1 | 0x02); // set sign bit

    emit_byte(op2 | encode(dst));

    emit_byte(imm32 & 0xFF);

  } else {

    emit_byte(op1);

    emit_byte(op2 | encode(dst));

    emit_long(imm32);

  }

}

// immediate-to-memory forms

void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {

  assert((op1 & 0x01) == 1, "should be 32bit operation");

  assert((op1 & 0x02) == 0, "sign-extension bit should not be set");

  if (is8bit(imm32)) {

    emit_byte(op1 | 0x02); // set sign bit

    emit_operand(rm, adr, 1);

    emit_byte(imm32 & 0xFF);

  } else {

    emit_byte(op1);

    emit_operand(rm, adr, 4);

    emit_long(imm32);

  }

}

void Assembler::emit_arith(int op1, int op2, Register dst, jobject obj) {

  LP64_ONLY(ShouldNotReachHere());

  assert(isByte(op1) && isByte(op2), "wrong opcode");

  assert((op1 & 0x01) == 1, "should be 32bit operation");

  assert((op1 & 0x02) == 0, "sign-extension bit should not be set");

  InstructionMark im(this);

  emit_byte(op1);

  emit_byte(op2 | encode(dst));

  emit_data((intptr_t)obj, relocInfo::oop_type, 0);

}

void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {

  assert(isByte(op1) && isByte(op2), "wrong opcode");

  emit_byte(op1);

  emit_byte(op2 | encode(dst) << 3 | encode(src));

}

void Assembler::emit_operand(Register reg, Register base, Register index,

                             Address::ScaleFactor scale, int disp,

                             RelocationHolder const& rspec,

                             int rip_relative_correction) {

  relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();

  // Encode the registers as needed in the fields they are used in

  int regenc = encode(reg) << 3;

  int indexenc = index->is_valid() ? encode(index) << 3 : 0;

  int baseenc = base->is_valid() ? encode(base) : 0;

  if (base->is_valid()) {

    if (index->is_valid()) {

      assert(scale != Address::no_scale, "inconsistent address");

      // [base + index*scale + disp]

      if (disp == 0 && rtype == relocInfo::none  &&

          base != rbp LP64_ONLY(&& base != r13)) {

        // [base + index*scale]

        // [00 reg 100][ss index base]

        assert(index != rsp, "illegal addressing mode");

        emit_byte(0x04 | regenc);

        emit_byte(scale << 6 | indexenc | baseenc);

      } else if (is8bit(disp) && rtype == relocInfo::none) {

        // [base + index*scale + imm8]

        // [01 reg 100][ss index base] imm8

        assert(index != rsp, "illegal addressing mode");

        emit_byte(0x44 | regenc);

        emit_byte(scale << 6 | indexenc | baseenc);

        emit_byte(disp & 0xFF);

      } else {

        // [base + index*scale + disp32]

        // [10 reg 100][ss index base] disp32

        assert(index != rsp, "illegal addressing mode");

        emit_byte(0x84 | regenc);

        emit_byte(scale << 6 | indexenc | baseenc);

        emit_data(disp, rspec, disp32_operand);

      }

    } else if (base == rsp LP64_ONLY(|| base == r12)) {

      // [rsp + disp]

      if (disp == 0 && rtype == relocInfo::none) {

        // [rsp]

        // [00 reg 100][00 100 100]

        emit_byte(0x04 | regenc);

        emit_byte(0x24);

      } else if (is8bit(disp) && rtype == relocInfo::none) {

        // [rsp + imm8]

        // [01 reg 100][00 100 100] disp8

        emit_byte(0x44 | regenc);

        emit_byte(0x24);

        emit_byte(disp & 0xFF);

      } else {

        // [rsp + imm32]

        // [10 reg 100][00 100 100] disp32

        emit_byte(0x84 | regenc);

        emit_byte(0x24);

        emit_data(disp, rspec, disp32_operand);

      }

    } else {

      // [base + disp]

      assert(base != rsp LP64_ONLY(&& base != r12), "illegal addressing mode");

      if (disp == 0 && rtype == relocInfo::none &&

          base != rbp LP64_ONLY(&& base != r13)) {

        // [base]

        // [00 reg base]

        emit_byte(0x00 | regenc | baseenc);

      } else if (is8bit(disp) && rtype == relocInfo::none) {

        // [base + disp8]

        // [01 reg base] disp8

        emit_byte(0x40 | regenc | baseenc);

        emit_byte(disp & 0xFF);

      } else {

        // [base + disp32]

        // [10 reg base] disp32

        emit_byte(0x80 | regenc | baseenc);

        emit_data(disp, rspec, disp32_operand);

      }

    }

  } else {

    if (index->is_valid()) {

      assert(scale != Address::no_scale, "inconsistent address");

      // [index*scale + disp]

      // [00 reg 100][ss index 101] disp32

      assert(index != rsp, "illegal addressing mode");

      emit_byte(0x04 | regenc);

      emit_byte(scale << 6 | indexenc | 0x05);

      emit_data(disp, rspec, disp32_operand);

    } else if (rtype != relocInfo::none ) {

      // [disp] (64bit) RIP-RELATIVE (32bit) abs

      // [00 000 101] disp32

      emit_byte(0x05 | regenc);

      // Note that the RIP-rel. correction applies to the generated

      // disp field, but _not_ to the target address in the rspec.

      // disp was created by converting the target address minus the pc

      // at the start of the instruction. That needs more correction here.

      // intptr_t disp = target - next_ip;

      assert(inst_mark() != NULL, "must be inside InstructionMark");

      address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;

      int64_t adjusted = disp;

      // Do rip-rel adjustment for 64bit

      LP64_ONLY(adjusted -=  (next_ip - inst_mark()));

      assert(is_simm32(adjusted),

             "must be 32bit offset (RIP relative address)");

      emit_data((int32_t) adjusted, rspec, disp32_operand);

    } else {

      // 32bit never did this, did everything as the rip-rel/disp code above

      // [disp] ABSOLUTE

      // [00 reg 100][00 100 101] disp32

      emit_byte(0x04 | regenc);

      emit_byte(0x25);

      emit_data(disp, rspec, disp32_operand);

    }

  }

}

void Assembler::emit_operand(XMMRegister reg, Register base, Register index,

                             Address::ScaleFactor scale, int disp,

                             RelocationHolder const& rspec) {

  emit_operand((Register)reg, base, index, scale, disp, rspec);

}

// Secret local extension to Assembler::WhichOperand:

#define end_pc_operand (_WhichOperand_limit)

address Assembler::locate_operand(address inst, WhichOperand which) {

  // Decode the given instruction, and return the address of

  // an embedded 32-bit operand word.

  // If "which" is disp32_operand, selects the displacement portion

  // of an effective address specifier.

  // If "which" is imm64_operand, selects the trailing immediate constant.

  // If "which" is call32_operand, selects the displacement of a call or jump.

  // Caller is responsible for ensuring that there is such an operand,

  // and that it is 32/64 bits wide.

  // If "which" is end_pc_operand, find the end of the instruction.

  address ip = inst;

  bool is_64bit = false;

  debug_only(bool has_disp32 = false);

  int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn

  again_after_prefix:

  switch (0xFF & *ip++) {

  // These convenience macros generate groups of "case" labels for the switch.

#define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3

#define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \

             case (x)+4: case (x)+5: case (x)+6: case (x)+7

#define REP16(x) REP8((x)+0): \

              case REP8((x)+8)

  case CS_segment:

  case SS_segment:

  case DS_segment:

  case ES_segment:

  case FS_segment:

  case GS_segment:

    // Seems dubious

    LP64_ONLY(assert(false, "shouldn't have that prefix"));

    assert(ip == inst+1, "only one prefix allowed");

    goto again_after_prefix;

  case 0x67:

  case REX:

  case REX_B:

  case REX_X:

  case REX_XB:

  case REX_R:

  case REX_RB:

  case REX_RX:

  case REX_RXB:

    NOT_LP64(assert(false, "64bit prefixes"));

    goto again_after_prefix;

  case REX_W:

  case REX_WB:

  case REX_WX:

  case REX_WXB:

  case REX_WR:

  case REX_WRB:

  case REX_WRX:

  case REX_WRXB:

    NOT_LP64(assert(false, "64bit prefixes"));

    is_64bit = true;

    goto again_after_prefix;

  case 0xFF: // pushq a; decl a; incl a; call a; jmp a

  case 0x88: // movb a, r

  case 0x89: // movl a, r

  case 0x8A: // movb r, a

  case 0x8B: // movl r, a

  case 0x8F: // popl a

    debug_only(has_disp32 = true);

    break;

  case 0x68: // pushq #32

    if (which == end_pc_operand) {

      return ip + 4;

    }

    assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");

    return ip;                  // not produced by emit_operand

  case 0x66: // movw ... (size prefix)

    again_after_size_prefix2:

    switch (0xFF & *ip++) {

    case REX:

    case REX_B:

    case REX_X:

    case REX_XB:

    case REX_R:

    case REX_RB:

    case REX_RX:

    case REX_RXB:

    case REX_W:

    case REX_WB:

    case REX_WX:

    case REX_WXB:

    case REX_WR:

    case REX_WRB:

    case REX_WRX:

    case REX_WRXB:

      NOT_LP64(assert(false, "64bit prefix found"));

      goto again_after_size_prefix2;

    case 0x8B: // movw r, a

    case 0x89: // movw a, r

      debug_only(has_disp32 = true);

      break;

    case 0xC7: // movw a, #16