--- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp Wed Apr 11 10:05:02 2018 -0400
+++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp Wed Apr 11 16:07:42 2018 +0200
@@ -27,12 +27,13 @@
#include "asm/assembler.hpp"
#include "asm/assembler.inline.hpp"
#include "compiler/disassembler.hpp"
-#include "gc/shared/cardTable.hpp"
-#include "gc/shared/cardTableBarrierSet.hpp"
+#include "gc/shared/barrierSet.hpp"
+#include "gc/shared/barrierSetAssembler.hpp"
#include "gc/shared/collectedHeap.inline.hpp"
#include "interpreter/interpreter.hpp"
#include "memory/resourceArea.hpp"
#include "memory/universe.hpp"
+#include "oops/access.hpp"
#include "oops/klass.inline.hpp"
#include "prims/methodHandles.hpp"
#include "runtime/biasedLocking.hpp"
@@ -45,11 +46,6 @@
#include "runtime/stubRoutines.hpp"
#include "runtime/thread.hpp"
#include "utilities/macros.hpp"
-#if INCLUDE_ALL_GCS
-#include "gc/g1/g1BarrierSet.hpp"
-#include "gc/g1/g1CardTable.hpp"
-#include "gc/g1/heapRegion.hpp"
-#endif // INCLUDE_ALL_GCS
#include "crc32c.h"
#ifdef COMPILER2
#include "opto/intrinsicnode.hpp"
@@ -5239,6 +5235,12 @@
}
}
+void MacroAssembler::clear_jweak_tag(Register possibly_jweak) {
+ const int32_t inverted_jweak_mask = ~static_cast<int32_t>(JNIHandles::weak_tag_mask);
+ STATIC_ASSERT(inverted_jweak_mask == -2); // otherwise check this code
+ // The inverted mask is sign-extended
+ andptr(possibly_jweak, inverted_jweak_mask);
+}
void MacroAssembler::resolve_jobject(Register value,
Register thread,
@@ -5250,296 +5252,18 @@
testptr(value, JNIHandles::weak_tag_mask); // Test for jweak tag.
jcc(Assembler::zero, not_weak);
// Resolve jweak.
- movptr(value, Address(value, -JNIHandles::weak_tag_value));
+ access_load_at(T_OBJECT, IN_ROOT | ON_PHANTOM_OOP_REF,
+ value, Address(value, -JNIHandles::weak_tag_value), tmp, thread);
verify_oop(value);
-#if INCLUDE_ALL_GCS
- if (UseG1GC) {
- g1_write_barrier_pre(noreg /* obj */,
- value /* pre_val */,
- thread /* thread */,
- tmp /* tmp */,
- true /* tosca_live */,
- true /* expand_call */);
- }
-#endif // INCLUDE_ALL_GCS
jmp(done);
bind(not_weak);
// Resolve (untagged) jobject.
- movptr(value, Address(value, 0));
+ access_load_at(T_OBJECT, IN_ROOT | IN_CONCURRENT_ROOT,
+ value, Address(value, 0), tmp, thread);
verify_oop(value);
bind(done);
}
-void MacroAssembler::clear_jweak_tag(Register possibly_jweak) {
- const int32_t inverted_jweak_mask = ~static_cast<int32_t>(JNIHandles::weak_tag_mask);
- STATIC_ASSERT(inverted_jweak_mask == -2); // otherwise check this code
- // The inverted mask is sign-extended
- andptr(possibly_jweak, inverted_jweak_mask);
-}
-
-//////////////////////////////////////////////////////////////////////////////////
-#if INCLUDE_ALL_GCS
-
-void MacroAssembler::g1_write_barrier_pre(Register obj,
- Register pre_val,
- Register thread,
- Register tmp,
- bool tosca_live,
- bool expand_call) {
-
- // If expand_call is true then we expand the call_VM_leaf macro
- // directly to skip generating the check by
- // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
-
-#ifdef _LP64
- assert(thread == r15_thread, "must be");
-#endif // _LP64
-
- Label done;
- Label runtime;
-
- assert(pre_val != noreg, "check this code");
-
- if (obj != noreg) {
- assert_different_registers(obj, pre_val, tmp);
- assert(pre_val != rax, "check this code");
- }
-
- Address in_progress(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
- SATBMarkQueue::byte_offset_of_active()));
- Address index(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
- SATBMarkQueue::byte_offset_of_index()));
- Address buffer(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
- SATBMarkQueue::byte_offset_of_buf()));
-
-
- // Is marking active?
- if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
- cmpl(in_progress, 0);
- } else {
- assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
- cmpb(in_progress, 0);
- }
- jcc(Assembler::equal, done);
-
- // Do we need to load the previous value?
- if (obj != noreg) {
- load_heap_oop(pre_val, Address(obj, 0));
- }
-
- // Is the previous value null?
- cmpptr(pre_val, (int32_t) NULL_WORD);
- jcc(Assembler::equal, done);
-
- // Can we store original value in the thread's buffer?
- // Is index == 0?
- // (The index field is typed as size_t.)
-
- movptr(tmp, index); // tmp := *index_adr
- cmpptr(tmp, 0); // tmp == 0?
- jcc(Assembler::equal, runtime); // If yes, goto runtime
-
- subptr(tmp, wordSize); // tmp := tmp - wordSize
- movptr(index, tmp); // *index_adr := tmp
- addptr(tmp, buffer); // tmp := tmp + *buffer_adr
-
- // Record the previous value
- movptr(Address(tmp, 0), pre_val);
- jmp(done);
-
- bind(runtime);
- // save the live input values
- if(tosca_live) push(rax);
-
- if (obj != noreg && obj != rax)
- push(obj);
-
- if (pre_val != rax)
- push(pre_val);
-
- // Calling the runtime using the regular call_VM_leaf mechanism generates
- // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
- // that checks that the *(ebp+frame::interpreter_frame_last_sp) == NULL.
- //
- // If we care generating the pre-barrier without a frame (e.g. in the
- // intrinsified Reference.get() routine) then ebp might be pointing to
- // the caller frame and so this check will most likely fail at runtime.
- //
- // Expanding the call directly bypasses the generation of the check.
- // So when we do not have have a full interpreter frame on the stack
- // expand_call should be passed true.
-
- NOT_LP64( push(thread); )
-
- if (expand_call) {
- LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
- pass_arg1(this, thread);
- pass_arg0(this, pre_val);
- MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), 2);
- } else {
- call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), pre_val, thread);
- }
-
- NOT_LP64( pop(thread); )
-
- // save the live input values
- if (pre_val != rax)
- pop(pre_val);
-
- if (obj != noreg && obj != rax)
- pop(obj);
-
- if(tosca_live) pop(rax);
-
- bind(done);
-}
-
-void MacroAssembler::g1_write_barrier_post(Register store_addr,
- Register new_val,
- Register thread,
- Register tmp,
- Register tmp2) {
-#ifdef _LP64
- assert(thread == r15_thread, "must be");
-#endif // _LP64
-
- Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
- DirtyCardQueue::byte_offset_of_index()));
- Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
- DirtyCardQueue::byte_offset_of_buf()));
-
- CardTableBarrierSet* ctbs =
- barrier_set_cast<CardTableBarrierSet>(Universe::heap()->barrier_set());
- CardTable* ct = ctbs->card_table();
- assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code");
-
- Label done;
- Label runtime;
-
- // Does store cross heap regions?
-
- movptr(tmp, store_addr);
- xorptr(tmp, new_val);
- shrptr(tmp, HeapRegion::LogOfHRGrainBytes);
- jcc(Assembler::equal, done);
-
- // crosses regions, storing NULL?
-
- cmpptr(new_val, (int32_t) NULL_WORD);
- jcc(Assembler::equal, done);
-
- // storing region crossing non-NULL, is card already dirty?
-
- const Register card_addr = tmp;
- const Register cardtable = tmp2;
-
- movptr(card_addr, store_addr);
- shrptr(card_addr, CardTable::card_shift);
- // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
- // a valid address and therefore is not properly handled by the relocation code.
- movptr(cardtable, (intptr_t)ct->byte_map_base());
- addptr(card_addr, cardtable);
-
- cmpb(Address(card_addr, 0), (int)G1CardTable::g1_young_card_val());
- jcc(Assembler::equal, done);
-
- membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
- cmpb(Address(card_addr, 0), (int)CardTable::dirty_card_val());
- jcc(Assembler::equal, done);
-
-
- // storing a region crossing, non-NULL oop, card is clean.
- // dirty card and log.
-
- movb(Address(card_addr, 0), (int)CardTable::dirty_card_val());
-
- cmpl(queue_index, 0);
- jcc(Assembler::equal, runtime);
- subl(queue_index, wordSize);
- movptr(tmp2, buffer);
-#ifdef _LP64
- movslq(rscratch1, queue_index);
- addq(tmp2, rscratch1);
- movq(Address(tmp2, 0), card_addr);
-#else
- addl(tmp2, queue_index);
- movl(Address(tmp2, 0), card_addr);
-#endif
- jmp(done);
-
- bind(runtime);
- // save the live input values
- push(store_addr);
- push(new_val);
-#ifdef _LP64
- call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, r15_thread);
-#else
- push(thread);
- call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, thread);
- pop(thread);
-#endif
- pop(new_val);
- pop(store_addr);
-
- bind(done);
-}
-
-#endif // INCLUDE_ALL_GCS
-//////////////////////////////////////////////////////////////////////////////////
-
-
-void MacroAssembler::store_check(Register obj, Address dst) {
- store_check(obj);
-}
-
-void MacroAssembler::store_check(Register obj) {
- // Does a store check for the oop in register obj. The content of
- // register obj is destroyed afterwards.
- BarrierSet* bs = Universe::heap()->barrier_set();
- assert(bs->kind() == BarrierSet::CardTableBarrierSet,
- "Wrong barrier set kind");
-
- CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs);
- CardTable* ct = ctbs->card_table();
- assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code");
-
- shrptr(obj, CardTable::card_shift);
-
- Address card_addr;
-
- // The calculation for byte_map_base is as follows:
- // byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift);
- // So this essentially converts an address to a displacement and it will
- // never need to be relocated. On 64bit however the value may be too
- // large for a 32bit displacement.
- intptr_t disp = (intptr_t) ct->byte_map_base();
- if (is_simm32(disp)) {
- card_addr = Address(noreg, obj, Address::times_1, disp);
- } else {
- // By doing it as an ExternalAddress 'disp' could be converted to a rip-relative
- // displacement and done in a single instruction given favorable mapping and a
- // smarter version of as_Address. However, 'ExternalAddress' generates a relocation
- // entry and that entry is not properly handled by the relocation code.
- AddressLiteral cardtable((address)ct->byte_map_base(), relocInfo::none);
- Address index(noreg, obj, Address::times_1);
- card_addr = as_Address(ArrayAddress(cardtable, index));
- }
-
- int dirty = CardTable::dirty_card_val();
- if (UseCondCardMark) {
- Label L_already_dirty;
- if (UseConcMarkSweepGC) {
- membar(Assembler::StoreLoad);
- }
- cmpb(card_addr, dirty);
- jcc(Assembler::equal, L_already_dirty);
- movb(card_addr, dirty);
- bind(L_already_dirty);
- } else {
- movb(card_addr, dirty);
- }
-}
-
void MacroAssembler::subptr(Register dst, int32_t imm32) {
LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32));
}
@@ -6590,50 +6314,47 @@
movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
}
-void MacroAssembler::load_heap_oop(Register dst, Address src) {
-#ifdef _LP64
- // FIXME: Must change all places where we try to load the klass.
- if (UseCompressedOops) {
- movl(dst, src);
- decode_heap_oop(dst);
- } else
-#endif
- movptr(dst, src);
+void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
+ Register tmp1, Register thread_tmp) {
+ BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
+ bool as_raw = (decorators & AS_RAW) != 0;
+ if (as_raw) {
+ bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
+ } else {
+ bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
+ }
+}
+
+void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register src,
+ Register tmp1, Register tmp2) {
+ BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
+ bool as_raw = (decorators & AS_RAW) != 0;
+ if (as_raw) {
+ bs->BarrierSetAssembler::store_at(this, decorators, type, dst, src, tmp1, tmp2);
+ } else {
+ bs->store_at(this, decorators, type, dst, src, tmp1, tmp2);
+ }
+}
+
+void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
+ Register thread_tmp, DecoratorSet decorators) {
+ access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp);
}
// Doesn't do verfication, generates fixed size code
-void MacroAssembler::load_heap_oop_not_null(Register dst, Address src) {
-#ifdef _LP64
- if (UseCompressedOops) {
- movl(dst, src);
- decode_heap_oop_not_null(dst);
- } else
-#endif
- movptr(dst, src);
-}
-
-void MacroAssembler::store_heap_oop(Address dst, Register src) {
-#ifdef _LP64
- if (UseCompressedOops) {
- assert(!dst.uses(src), "not enough registers");
- encode_heap_oop(src);
- movl(dst, src);
- } else
-#endif
- movptr(dst, src);
+void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
+ Register thread_tmp, DecoratorSet decorators) {
+ access_load_at(T_OBJECT, IN_HEAP | OOP_NOT_NULL | decorators, dst, src, tmp1, thread_tmp);
+}
+
+void MacroAssembler::store_heap_oop(Address dst, Register src, Register tmp1,
+ Register tmp2, DecoratorSet decorators) {
+ access_store_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, tmp2);
}
// Used for storing NULLs.
void MacroAssembler::store_heap_oop_null(Address dst) {
-#ifdef _LP64
- if (UseCompressedOops) {
- movl(dst, (int32_t)NULL_WORD);
- } else {
- movslq(dst, (int32_t)NULL_WORD);
- }
-#else
- movl(dst, (int32_t)NULL_WORD);
-#endif
+ access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg);
}
#ifdef _LP64