--- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp Wed Oct 16 11:12:47 2019 +0100
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp Fri Oct 18 13:07:46 2019 -0400
@@ -333,7 +333,7 @@
#endif
}
-void ShenandoahBarrierSetAssembler::load_reference_barrier_native(MacroAssembler* masm, Register dst) {
+void ShenandoahBarrierSetAssembler::load_reference_barrier_native(MacroAssembler* masm, Register dst, Address src) {
if (!ShenandoahLoadRefBarrier) {
return;
}
@@ -341,6 +341,7 @@
Label done;
Label not_null;
Label slow_path;
+ __ block_comment("load_reference_barrier_native { ");
// null check
__ testptr(dst, dst);
@@ -371,7 +372,7 @@
__ bind(slow_path);
if (dst != rax) {
- __ xchgptr(dst, rax); // Move obj into rax and save rax into obj.
+ __ push(rax);
}
__ push(rcx);
__ push(rdx);
@@ -388,8 +389,9 @@
__ push(r15);
#endif
- __ movptr(rdi, rax);
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native), rdi);
+ assert_different_registers(dst, rsi);
+ __ lea(rsi, src);
+ __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native), dst, rsi);
#ifdef _LP64
__ pop(r15);
@@ -407,10 +409,12 @@
__ pop(rcx);
if (dst != rax) {
- __ xchgptr(rax, dst); // Swap back obj with rax.
+ __ movptr(dst, rax);
+ __ pop(rax);
}
__ bind(done);
+ __ block_comment("load_reference_barrier_native { ");
}
void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
@@ -474,14 +478,43 @@
bool is_traversal_mode = ShenandoahHeap::heap()->is_traversal_mode();
bool keep_alive = ((decorators & AS_NO_KEEPALIVE) == 0) || is_traversal_mode;
+ Register result_dst = dst;
+ bool use_tmp1_for_dst = false;
+
+ if (on_oop) {
+ // We want to preserve src
+ if (dst == src.base() || dst == src.index()) {
+ // Use tmp1 for dst if possible, as it is not used in BarrierAssembler::load_at()
+ if (tmp1->is_valid() && tmp1 != src.base() && tmp1 != src.index()) {
+ dst = tmp1;
+ use_tmp1_for_dst = true;
+ } else {
+ dst = rdi;
+ __ push(dst);
+ }
+ }
+ assert_different_registers(dst, src.base(), src.index());
+ }
+
BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
+
if (on_oop) {
if (not_in_heap && !is_traversal_mode) {
- load_reference_barrier_native(masm, dst);
+ load_reference_barrier_native(masm, dst, src);
} else {
load_reference_barrier(masm, dst);
}
+ if (dst != result_dst) {
+ __ movptr(result_dst, dst);
+
+ if (!use_tmp1_for_dst) {
+ __ pop(dst);
+ }
+
+ dst = result_dst;
+ }
+
if (ShenandoahKeepAliveBarrier && on_reference && keep_alive) {
const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
assert_different_registers(dst, tmp1, tmp_thread);
--- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp Wed Oct 16 11:12:47 2019 +0100
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp Fri Oct 18 13:07:46 2019 -0400
@@ -76,7 +76,7 @@
#endif
void load_reference_barrier(MacroAssembler* masm, Register dst);
- void load_reference_barrier_native(MacroAssembler* masm, Register dst);
+ void load_reference_barrier_native(MacroAssembler* masm, Register dst, Address src);
void cmpxchg_oop(MacroAssembler* masm,
Register res, Address addr, Register oldval, Register newval,
--- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp Wed Oct 16 11:12:47 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp Fri Oct 18 13:07:46 2019 -0400
@@ -215,9 +215,13 @@
assert(access.is_oop(), "IN_NATIVE access only for oop values");
BarrierSetC1::load_at_resolved(access, result);
LIR_OprList* args = new LIR_OprList();
+ LIR_Opr addr = access.resolved_addr();
+ addr = ensure_in_register(gen, addr);
args->append(result);
+ args->append(addr);
BasicTypeList signature;
signature.append(T_OBJECT);
+ signature.append(T_ADDRESS);
LIR_Opr call_result = gen->call_runtime(&signature, args,
CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native),
objectType, NULL);
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp Wed Oct 16 11:12:47 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp Fri Oct 18 13:07:46 2019 -0400
@@ -262,7 +262,7 @@
}
}
-oop ShenandoahBarrierSet::oop_load_from_native_barrier(oop obj) {
+oop ShenandoahBarrierSet::oop_load_from_native_barrier(oop obj, oop* load_addr) {
if (CompressedOops::is_null(obj)) {
return NULL;
}
@@ -277,7 +277,13 @@
}
}
- return load_reference_barrier_not_null(obj);
+ oop fwd = load_reference_barrier_not_null(obj);
+ if (load_addr != NULL && fwd != obj) {
+ // Since we are here and we know the load address, update the reference.
+ ShenandoahHeap::cas_oop(fwd, load_addr, obj);
+ }
+
+ return fwd;
}
void ShenandoahBarrierSet::clone_barrier_runtime(oop src) {
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp Wed Oct 16 11:12:47 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp Fri Oct 18 13:07:46 2019 -0400
@@ -84,7 +84,9 @@
void write_ref_field_work(void* v, oop o, bool release = false);
- oop oop_load_from_native_barrier(oop obj);
+ oop oop_load_from_native_barrier(oop obj, oop* load_addr);
+ oop oop_load_from_native_barrier(oop obj, narrowOop* load_addr);
+
virtual void on_thread_create(Thread* thread);
virtual void on_thread_destroy(Thread* thread);
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp Wed Oct 16 11:12:47 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp Fri Oct 18 13:07:46 2019 -0400
@@ -69,7 +69,7 @@
template <typename T>
inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(T* addr) {
oop value = Raw::oop_load_not_in_heap(addr);
- value = ShenandoahBarrierSet::barrier_set()->oop_load_from_native_barrier(value);
+ value = ShenandoahBarrierSet::barrier_set()->oop_load_from_native_barrier(value, addr);
keep_alive_if_weak(decorators, value);
return value;
}
--- a/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp Wed Oct 16 11:12:47 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp Fri Oct 18 13:07:46 2019 -0400
@@ -82,6 +82,6 @@
ShenandoahBarrierSet::barrier_set()->clone_barrier(s);
JRT_END
-JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_native(oopDesc * src))
- return (oopDesc*) ShenandoahBarrierSet::barrier_set()->oop_load_from_native_barrier(oop(src));
+JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_native(oopDesc * src, oop* load_addr))
+ return (oopDesc*) ShenandoahBarrierSet::barrier_set()->oop_load_from_native_barrier(oop(src), load_addr);
JRT_END
--- a/src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp Wed Oct 16 11:12:47 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp Fri Oct 18 13:07:46 2019 -0400
@@ -42,7 +42,7 @@
static oopDesc* load_reference_barrier_fixup(oopDesc* src, oop* load_addr);
static oopDesc* load_reference_barrier_fixup_narrow(oopDesc* src, narrowOop* load_addr);
- static oopDesc* load_reference_barrier_native(oopDesc* src);
+ static oopDesc* load_reference_barrier_native(oopDesc* src, oop* load_addr);
static void shenandoah_clone_barrier(oopDesc* src);
};