--- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp Mon Aug 12 20:27:14 2019 +0200
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp Mon Aug 12 23:37:48 2019 +0200
@@ -752,108 +752,6 @@
}
}
-void ShenandoahBarrierSetAssembler::save_vector_registers(MacroAssembler* masm) {
- int num_xmm_regs = LP64_ONLY(16) NOT_LP64(8);
- if (UseAVX > 2) {
- num_xmm_regs = LP64_ONLY(32) NOT_LP64(8);
- }
-
- if (UseSSE == 1) {
- __ subptr(rsp, sizeof(jdouble)*8);
- for (int n = 0; n < 8; n++) {
- __ movflt(Address(rsp, n*sizeof(jdouble)), as_XMMRegister(n));
- }
- } else if (UseSSE >= 2) {
- if (UseAVX > 2) {
- __ push(rbx);
- __ movl(rbx, 0xffff);
- __ kmovwl(k1, rbx);
- __ pop(rbx);
- }
-#ifdef COMPILER2
- if (MaxVectorSize > 16) {
- if(UseAVX > 2) {
- // Save upper half of ZMM registers
- __ subptr(rsp, 32*num_xmm_regs);
- for (int n = 0; n < num_xmm_regs; n++) {
- __ vextractf64x4_high(Address(rsp, n*32), as_XMMRegister(n));
- }
- }
- assert(UseAVX > 0, "256 bit vectors are supported only with AVX");
- // Save upper half of YMM registers
- __ subptr(rsp, 16*num_xmm_regs);
- for (int n = 0; n < num_xmm_regs; n++) {
- __ vextractf128_high(Address(rsp, n*16), as_XMMRegister(n));
- }
- }
-#endif
- // Save whole 128bit (16 bytes) XMM registers
- __ subptr(rsp, 16*num_xmm_regs);
-#ifdef _LP64
- if (VM_Version::supports_evex()) {
- for (int n = 0; n < num_xmm_regs; n++) {
- __ vextractf32x4(Address(rsp, n*16), as_XMMRegister(n), 0);
- }
- } else {
- for (int n = 0; n < num_xmm_regs; n++) {
- __ movdqu(Address(rsp, n*16), as_XMMRegister(n));
- }
- }
-#else
- for (int n = 0; n < num_xmm_regs; n++) {
- __ movdqu(Address(rsp, n*16), as_XMMRegister(n));
- }
-#endif
- }
-}
-
-void ShenandoahBarrierSetAssembler::restore_vector_registers(MacroAssembler* masm) {
- int num_xmm_regs = LP64_ONLY(16) NOT_LP64(8);
- if (UseAVX > 2) {
- num_xmm_regs = LP64_ONLY(32) NOT_LP64(8);
- }
- if (UseSSE == 1) {
- for (int n = 0; n < 8; n++) {
- __ movflt(as_XMMRegister(n), Address(rsp, n*sizeof(jdouble)));
- }
- __ addptr(rsp, sizeof(jdouble)*8);
- } else if (UseSSE >= 2) {
- // Restore whole 128bit (16 bytes) XMM registers
-#ifdef _LP64
- if (VM_Version::supports_evex()) {
- for (int n = 0; n < num_xmm_regs; n++) {
- __ vinsertf32x4(as_XMMRegister(n), as_XMMRegister(n), Address(rsp, n*16), 0);
- }
- } else {
- for (int n = 0; n < num_xmm_regs; n++) {
- __ movdqu(as_XMMRegister(n), Address(rsp, n*16));
- }
- }
-#else
- for (int n = 0; n < num_xmm_regs; n++) {
- __ movdqu(as_XMMRegister(n), Address(rsp, n*16));
- }
-#endif
- __ addptr(rsp, 16*num_xmm_regs);
-
-#ifdef COMPILER2
- if (MaxVectorSize > 16) {
- // Restore upper half of YMM registers.
- for (int n = 0; n < num_xmm_regs; n++) {
- __ vinsertf128_high(as_XMMRegister(n), Address(rsp, n*16));
- }
- __ addptr(rsp, 16*num_xmm_regs);
- if (UseAVX > 2) {
- for (int n = 0; n < num_xmm_regs; n++) {
- __ vinsertf64x4_high(as_XMMRegister(n), Address(rsp, n*32));
- }
- __ addptr(rsp, 32*num_xmm_regs);
- }
- }
-#endif
- }
-}
-
#undef __
#ifdef COMPILER1
@@ -885,11 +783,17 @@
}
void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
+ ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
__ bind(*stub->entry());
- Label done;
Register obj = stub->obj()->as_register();
Register res = stub->result()->as_register();
+ Register tmp1 = stub->tmp1()->as_register();
+ Register tmp2 = stub->tmp2()->as_register();
+
+ Label slow_path;
+
+ assert(res == rax, "result must arrive in rax");
if (res != obj) {
__ mov(res, obj);
@@ -897,11 +801,35 @@
// Check for null.
__ testptr(res, res);
- __ jcc(Assembler::zero, done);
+ __ jcc(Assembler::zero, *stub->continuation());
+
+ // Check for object being in the collection set.
+ __ mov(tmp1, res);
+ __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+ __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
+ __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1));
+ __ testbool(tmp2);
+ __ jcc(Assembler::zero, *stub->continuation());
- load_reference_barrier_not_null(ce->masm(), res);
+ // Test if object is resolved.
+ __ movptr(tmp1, Address(res, oopDesc::mark_offset_in_bytes()));
+ // Test if both lowest bits are set. We trick it by negating the bits
+ // then test for both bits clear.
+ __ notptr(tmp1);
+ __ testb(tmp1, markOopDesc::marked_value);
+ __ jccb(Assembler::notZero, slow_path);
+ // Clear both lower bits. It's still inverted, so set them, and then invert back.
+ __ orptr(tmp1, markOopDesc::marked_value);
+ __ notptr(tmp1);
+ // At this point, tmp1 contains the decoded forwarding pointer.
+ __ mov(res, tmp1);
- __ bind(done);
+ __ jmp(*stub->continuation());
+
+ __ bind(slow_path);
+ ce->store_parameter(res, 0);
+ __ call(RuntimeAddress(bs->load_reference_barrier_rt_code_blob()->code_begin()));
+
__ jmp(*stub->continuation());
}
@@ -965,6 +893,18 @@
__ epilogue();
}
+void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm) {
+ __ prologue("shenandoah_load_reference_barrier", false);
+ // arg0 : object to be resolved
+
+ __ save_live_registers_no_oop_map(true);
+ __ load_parameter(0, c_rarg0);
+ __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), c_rarg0);
+ __ restore_live_registers_except_rax(true);
+
+ __ epilogue();
+}
+
#undef __
#endif // COMPILER1
@@ -981,11 +921,11 @@
StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb");
address start = __ pc();
- Label resolve_oop, slow_path;
+ Label resolve_oop, slow_path, done;
// We use RDI, which also serves as argument register for slow call.
- // RAX always holds the src object ptr, except after the slow call and
- // the cmpxchg, then it holds the result. R8/RBX is used as temporary register.
+ // RAX always holds the src object ptr, except after the slow call,
+ // then it holds the result. R8/RBX is used as temporary register.
Register tmp1 = rdi;
Register tmp2 = LP64_ONLY(r8) NOT_LP64(rbx);
@@ -994,26 +934,18 @@
__ push(tmp2);
// Check for object being in the collection set.
- // TODO: Can we use only 1 register here?
- // The source object arrives here in rax.
- // live: rax
- // live: tmp1
__ mov(tmp1, rax);
__ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
- // live: tmp2
__ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
__ movbool(tmp2, Address(tmp2, tmp1, Address::times_1));
- // unlive: tmp1
__ testbool(tmp2);
- // unlive: tmp2
__ jccb(Assembler::notZero, resolve_oop);
-
__ pop(tmp2);
__ pop(tmp1);
__ ret(0);
+ // Test if object is already resolved.
__ bind(resolve_oop);
-
__ movptr(tmp2, Address(rax, oopDesc::mark_offset_in_bytes()));
// Test if both lowest bits are set. We trick it by negating the bits
// then test for both bits clear.
@@ -1026,6 +958,7 @@
// At this point, tmp2 contains the decoded forwarding pointer.
__ mov(rax, tmp2);
+ __ bind(done);
__ pop(tmp2);
__ pop(tmp1);
__ ret(0);
@@ -1047,10 +980,7 @@
__ push(r15);
#endif
- save_vector_registers(cgen->assembler());
- __ movptr(rdi, rax);
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), rdi);
- restore_vector_registers(cgen->assembler());
+ __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), rax);
#ifdef _LP64
__ pop(r15);