--- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp	Mon Aug 12 20:27:14 2019 +0200
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp	Mon Aug 12 23:37:48 2019 +0200
@@ -521,23 +521,45 @@
 }
 
 void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
+  ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
+  __ bind(*stub->entry());
 
   Register obj = stub->obj()->as_register();
   Register res = stub->result()->as_register();
+  Register tmp1 = stub->tmp1()->as_register();
+  Register tmp2 = stub->tmp2()->as_register();
 
-  Label done;
-
-  __ bind(*stub->entry());
+  assert(res == r0, "result must arrive in r0");
 
   if (res != obj) {
     __ mov(res, obj);
   }
+
   // Check for null.
-  __ cbz(res, done);
+  __ cbz(res, *stub->continuation());
+
+  // Check for object in cset.
+  __ mov(tmp2, ShenandoahHeap::in_cset_fast_test_addr());
+  __ lsr(tmp1, res, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+  __ ldrb(tmp2, Address(tmp2, tmp1));
+  __ cbz(tmp2, *stub->continuation());
 
-  load_reference_barrier_not_null(ce->masm(), res, rscratch1);
+  // Check if object is already forwarded.
+  Label slow_path;
+  __ ldr(tmp1, Address(res, oopDesc::mark_offset_in_bytes()));
+  __ eon(tmp1, tmp1, zr);
+  __ ands(zr, tmp1, markOopDesc::lock_mask_in_place);
+  __ br(Assembler::NE, slow_path);
 
-  __ bind(done);
+  // Decode forwarded object.
+  __ orr(tmp1, tmp1, markOopDesc::marked_value);
+  __ eon(res, tmp1, zr);
+  __ b(*stub->continuation());
+
+  __ bind(slow_path);
+  ce->store_parameter(res, 0);
+  __ far_call(RuntimeAddress(bs->load_reference_barrier_rt_code_blob()->code_begin()));
+
   __ b(*stub->continuation());
 }
 
@@ -591,6 +613,21 @@
   __ epilogue();
 }
 
+void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm) {
+  __ prologue("shenandoah_load_reference_barrier", false);
+  // arg0 : object to be resolved
+
+  __ push_call_clobbered_registers();
+  __ load_parameter(0, r0);
+  __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier));
+  __ blr(lr);
+  __ mov(rscratch1, r0);
+  __ pop_call_clobbered_registers();
+  __ mov(r0, rscratch1);
+
+  __ epilogue();
+}
+
 #undef __
 
 #endif // COMPILER1
@@ -625,11 +662,18 @@
   __ ret(lr);
   __ bind(work);
 
-  __ mov(rscratch2, r0);
-  resolve_forward_pointer_not_null(cgen->assembler(), r0, rscratch1);
-  __ cmp(rscratch2, r0);
-  __ br(Assembler::NE, done);
+  Label slow_path;
+  __ ldr(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes()));
+  __ eon(rscratch1, rscratch1, zr);
+  __ ands(zr, rscratch1, markOopDesc::lock_mask_in_place);
+  __ br(Assembler::NE, slow_path);
 
+  // Decode forwarded object.
+  __ orr(rscratch1, rscratch1, markOopDesc::marked_value);
+  __ eon(r0, rscratch1, zr);
+  __ ret(lr);
+
+  __ bind(slow_path);
   __ enter(); // required for proper stackwalking of RuntimeStub frame
 
   __ push_call_clobbered_registers();

--- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp	Mon Aug 12 20:27:14 2019 +0200
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp	Mon Aug 12 23:37:48 2019 +0200
@@ -71,6 +71,7 @@
   void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub);
   void gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub);
   void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
+  void generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm);
 #endif
 
   virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,

--- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp	Mon Aug 12 20:27:14 2019 +0200
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp	Mon Aug 12 23:37:48 2019 +0200
@@ -752,108 +752,6 @@
   }
 }
 
-void ShenandoahBarrierSetAssembler::save_vector_registers(MacroAssembler* masm) {
-  int num_xmm_regs = LP64_ONLY(16) NOT_LP64(8);
-  if (UseAVX > 2) {
-    num_xmm_regs = LP64_ONLY(32) NOT_LP64(8);
-  }
-
-  if (UseSSE == 1)  {
-    __ subptr(rsp, sizeof(jdouble)*8);
-    for (int n = 0; n < 8; n++) {
-      __ movflt(Address(rsp, n*sizeof(jdouble)), as_XMMRegister(n));
-    }
-  } else if (UseSSE >= 2)  {
-    if (UseAVX > 2) {
-      __ push(rbx);
-      __ movl(rbx, 0xffff);
-      __ kmovwl(k1, rbx);
-      __ pop(rbx);
-    }
-#ifdef COMPILER2
-    if (MaxVectorSize > 16) {
-      if(UseAVX > 2) {
-        // Save upper half of ZMM registers
-        __ subptr(rsp, 32*num_xmm_regs);
-        for (int n = 0; n < num_xmm_regs; n++) {
-          __ vextractf64x4_high(Address(rsp, n*32), as_XMMRegister(n));
-        }
-      }
-      assert(UseAVX > 0, "256 bit vectors are supported only with AVX");
-      // Save upper half of YMM registers
-      __ subptr(rsp, 16*num_xmm_regs);
-      for (int n = 0; n < num_xmm_regs; n++) {
-        __ vextractf128_high(Address(rsp, n*16), as_XMMRegister(n));
-      }
-    }
-#endif
-    // Save whole 128bit (16 bytes) XMM registers
-    __ subptr(rsp, 16*num_xmm_regs);
-#ifdef _LP64
-    if (VM_Version::supports_evex()) {
-      for (int n = 0; n < num_xmm_regs; n++) {
-        __ vextractf32x4(Address(rsp, n*16), as_XMMRegister(n), 0);
-      }
-    } else {
-      for (int n = 0; n < num_xmm_regs; n++) {
-        __ movdqu(Address(rsp, n*16), as_XMMRegister(n));
-      }
-    }
-#else
-    for (int n = 0; n < num_xmm_regs; n++) {
-      __ movdqu(Address(rsp, n*16), as_XMMRegister(n));
-    }
-#endif
-  }
-}
-
-void ShenandoahBarrierSetAssembler::restore_vector_registers(MacroAssembler* masm) {
-  int num_xmm_regs = LP64_ONLY(16) NOT_LP64(8);
-  if (UseAVX > 2) {
-    num_xmm_regs = LP64_ONLY(32) NOT_LP64(8);
-  }
-  if (UseSSE == 1)  {
-    for (int n = 0; n < 8; n++) {
-      __ movflt(as_XMMRegister(n), Address(rsp, n*sizeof(jdouble)));
-    }
-    __ addptr(rsp, sizeof(jdouble)*8);
-  } else if (UseSSE >= 2)  {
-    // Restore whole 128bit (16 bytes) XMM registers
-#ifdef _LP64
-    if (VM_Version::supports_evex()) {
-      for (int n = 0; n < num_xmm_regs; n++) {
-        __ vinsertf32x4(as_XMMRegister(n), as_XMMRegister(n), Address(rsp, n*16), 0);
-      }
-    } else {
-      for (int n = 0; n < num_xmm_regs; n++) {
-        __ movdqu(as_XMMRegister(n), Address(rsp, n*16));
-      }
-    }
-#else
-    for (int n = 0; n < num_xmm_regs; n++) {
-      __ movdqu(as_XMMRegister(n), Address(rsp, n*16));
-    }
-#endif
-    __ addptr(rsp, 16*num_xmm_regs);
-
-#ifdef COMPILER2
-    if (MaxVectorSize > 16) {
-      // Restore upper half of YMM registers.
-      for (int n = 0; n < num_xmm_regs; n++) {
-        __ vinsertf128_high(as_XMMRegister(n), Address(rsp, n*16));
-      }
-      __ addptr(rsp, 16*num_xmm_regs);
-      if (UseAVX > 2) {
-        for (int n = 0; n < num_xmm_regs; n++) {
-          __ vinsertf64x4_high(as_XMMRegister(n), Address(rsp, n*32));
-        }
-        __ addptr(rsp, 32*num_xmm_regs);
-      }
-    }
-#endif
-  }
-}
-
 #undef __
 
 #ifdef COMPILER1
@@ -885,11 +783,17 @@
 }
 
 void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
+  ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
   __ bind(*stub->entry());
 
-  Label done;
   Register obj = stub->obj()->as_register();
   Register res = stub->result()->as_register();
+  Register tmp1 = stub->tmp1()->as_register();
+  Register tmp2 = stub->tmp2()->as_register();
+
+  Label slow_path;
+
+  assert(res == rax, "result must arrive in rax");
 
   if (res != obj) {
     __ mov(res, obj);
@@ -897,11 +801,35 @@
 
   // Check for null.
   __ testptr(res, res);
-  __ jcc(Assembler::zero, done);
+  __ jcc(Assembler::zero, *stub->continuation());
+
+  // Check for object being in the collection set.
+  __ mov(tmp1, res);
+  __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+  __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
+  __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1));
+  __ testbool(tmp2);
+  __ jcc(Assembler::zero, *stub->continuation());
 
-  load_reference_barrier_not_null(ce->masm(), res);
+  // Test if object is resolved.
+  __ movptr(tmp1, Address(res, oopDesc::mark_offset_in_bytes()));
+  // Test if both lowest bits are set. We trick it by negating the bits
+  // then test for both bits clear.
+  __ notptr(tmp1);
+  __ testb(tmp1, markOopDesc::marked_value);
+  __ jccb(Assembler::notZero, slow_path);
+  // Clear both lower bits. It's still inverted, so set them, and then invert back.
+  __ orptr(tmp1, markOopDesc::marked_value);
+  __ notptr(tmp1);
+  // At this point, tmp1 contains the decoded forwarding pointer.
+  __ mov(res, tmp1);
 
-  __ bind(done);
+  __ jmp(*stub->continuation());
+
+  __ bind(slow_path);
+  ce->store_parameter(res, 0);
+  __ call(RuntimeAddress(bs->load_reference_barrier_rt_code_blob()->code_begin()));
+
   __ jmp(*stub->continuation());
 }
 
@@ -965,6 +893,18 @@
   __ epilogue();
 }
 
+void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm) {
+  __ prologue("shenandoah_load_reference_barrier", false);
+  // arg0 : object to be resolved
+
+  __ save_live_registers_no_oop_map(true);
+  __ load_parameter(0, c_rarg0);
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), c_rarg0);
+  __ restore_live_registers_except_rax(true);
+
+  __ epilogue();
+}
+
 #undef __
 
 #endif // COMPILER1
@@ -981,11 +921,11 @@
   StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb");
   address start = __ pc();
 
-  Label resolve_oop, slow_path;
+  Label resolve_oop, slow_path, done;
 
   // We use RDI, which also serves as argument register for slow call.
-  // RAX always holds the src object ptr, except after the slow call and
-  // the cmpxchg, then it holds the result. R8/RBX is used as temporary register.
+  // RAX always holds the src object ptr, except after the slow call,
+  // then it holds the result. R8/RBX is used as temporary register.
 
   Register tmp1 = rdi;
   Register tmp2 = LP64_ONLY(r8) NOT_LP64(rbx);
@@ -994,26 +934,18 @@
   __ push(tmp2);
 
   // Check for object being in the collection set.
-  // TODO: Can we use only 1 register here?
-  // The source object arrives here in rax.
-  // live: rax
-  // live: tmp1
   __ mov(tmp1, rax);
   __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
-  // live: tmp2
   __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
   __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1));
-  // unlive: tmp1
   __ testbool(tmp2);
-  // unlive: tmp2
   __ jccb(Assembler::notZero, resolve_oop);
-
   __ pop(tmp2);
   __ pop(tmp1);
   __ ret(0);
 
+  // Test if object is already resolved.
   __ bind(resolve_oop);
-
   __ movptr(tmp2, Address(rax, oopDesc::mark_offset_in_bytes()));
   // Test if both lowest bits are set. We trick it by negating the bits
   // then test for both bits clear.
@@ -1026,6 +958,7 @@
   // At this point, tmp2 contains the decoded forwarding pointer.
   __ mov(rax, tmp2);
 
+  __ bind(done);
   __ pop(tmp2);
   __ pop(tmp1);
   __ ret(0);
@@ -1047,10 +980,7 @@
   __ push(r15);
 #endif
 
-  save_vector_registers(cgen->assembler());
-  __ movptr(rdi, rax);
-  __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), rdi);
-  restore_vector_registers(cgen->assembler());
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), rax);
 
 #ifdef _LP64
   __ pop(r15);

--- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp	Mon Aug 12 20:27:14 2019 +0200
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp	Mon Aug 12 23:37:48 2019 +0200
@@ -64,9 +64,6 @@
 
   address generate_shenandoah_lrb(StubCodeGenerator* cgen);
 
-  void save_vector_registers(MacroAssembler* masm);
-  void restore_vector_registers(MacroAssembler* masm);
-
 public:
   static address shenandoah_lrb();
 
@@ -75,6 +72,7 @@
   void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub);
   void gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub);
   void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
+  void generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm);
 #endif
 
   void load_reference_barrier(MacroAssembler* masm, Register dst);

--- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp	Mon Aug 12 20:27:14 2019 +0200
+++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp	Mon Aug 12 23:37:48 2019 +0200
@@ -47,6 +47,10 @@
   bs->gen_load_reference_barrier_stub(ce, this);
 }
 
+ShenandoahBarrierSetC1::ShenandoahBarrierSetC1() :
+  _pre_barrier_c1_runtime_code_blob(NULL),
+  _load_reference_barrier_rt_code_blob(NULL) {}
+
 void ShenandoahBarrierSetC1::pre_barrier(LIRGenerator* gen, CodeEmitInfo* info, DecoratorSet decorators, LIR_Opr addr_opr, LIR_Opr pre_val) {
   // First we test whether marking is in progress.
   BasicType flag_type;
@@ -114,8 +118,10 @@
 
   obj = ensure_in_register(gen, obj);
   assert(obj->is_register(), "must be a register at this point");
-  LIR_Opr result = gen->new_register(T_OBJECT);
+  LIR_Opr result = gen->result_register_for(obj->value_type());
   __ move(obj, result);
+  LIR_Opr tmp1 = gen->new_register(T_OBJECT);
+  LIR_Opr tmp2 = gen->new_register(T_OBJECT);
 
   LIR_Opr thrd = gen->getThreadPointer();
   LIR_Address* active_flag_addr =
@@ -140,7 +146,7 @@
   }
   __ cmp(lir_cond_notEqual, flag_val, LIR_OprFact::intConst(0));
 
-  CodeStub* slow = new ShenandoahLoadReferenceBarrierStub(obj, result);
+  CodeStub* slow = new ShenandoahLoadReferenceBarrierStub(obj, result, tmp1, tmp2);
   __ branch(lir_cond_notEqual, T_INT, slow);
   __ branch_destination(slow->continuation());
 
@@ -238,11 +244,25 @@
   }
 };
 
+class C1ShenandoahLoadReferenceBarrierCodeGenClosure : public StubAssemblerCodeGenClosure {
+  virtual OopMapSet* generate_code(StubAssembler* sasm) {
+    ShenandoahBarrierSetAssembler* bs = (ShenandoahBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
+    bs->generate_c1_load_reference_barrier_runtime_stub(sasm);
+    return NULL;
+  }
+};
+
 void ShenandoahBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* buffer_blob) {
   C1ShenandoahPreBarrierCodeGenClosure pre_code_gen_cl;
   _pre_barrier_c1_runtime_code_blob = Runtime1::generate_blob(buffer_blob, -1,
                                                               "shenandoah_pre_barrier_slow",
                                                               false, &pre_code_gen_cl);
+  if (ShenandoahLoadRefBarrier) {
+    C1ShenandoahLoadReferenceBarrierCodeGenClosure lrb_code_gen_cl;
+    _load_reference_barrier_rt_code_blob = Runtime1::generate_blob(buffer_blob, -1,
+                                                                  "shenandoah_load_reference_barrier_slow",
+                                                                  false, &lrb_code_gen_cl);
+  }
 }
 
 const char* ShenandoahBarrierSetC1::rtcall_name_for_address(address entry) {

--- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp	Mon Aug 12 20:27:14 2019 +0200
+++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp	Mon Aug 12 23:37:48 2019 +0200
@@ -90,23 +90,31 @@
 private:
   LIR_Opr _obj;
   LIR_Opr _result;
+  LIR_Opr _tmp1;
+  LIR_Opr _tmp2;
 
 public:
-  ShenandoahLoadReferenceBarrierStub(LIR_Opr obj, LIR_Opr result) :
-    _obj(obj), _result(result)
+  ShenandoahLoadReferenceBarrierStub(LIR_Opr obj, LIR_Opr result, LIR_Opr tmp1, LIR_Opr tmp2) :
+    _obj(obj), _result(result), _tmp1(tmp1), _tmp2(tmp2)
   {
     assert(_obj->is_register(), "should be register");
     assert(_result->is_register(), "should be register");
+    assert(_tmp1->is_register(), "should be register");
+    assert(_tmp2->is_register(), "should be register");
   }
 
   LIR_Opr obj() const { return _obj; }
   LIR_Opr result() const { return _result; }
+  LIR_Opr tmp1() const { return _tmp1; }
+  LIR_Opr tmp2() const { return _tmp2; }
 
   virtual void emit_code(LIR_Assembler* e);
   virtual void visit(LIR_OpVisitState* visitor) {
     visitor->do_slow_case();
     visitor->do_input(_obj);
     visitor->do_temp(_result);
+    visitor->do_temp(_tmp1);
+    visitor->do_temp(_tmp2);
   }
 #ifndef PRODUCT
   virtual void print_name(outputStream* out) const { out->print("ShenandoahLoadReferenceBarrierStub"); }
@@ -174,6 +182,7 @@
 class ShenandoahBarrierSetC1 : public BarrierSetC1 {
 private:
   CodeBlob* _pre_barrier_c1_runtime_code_blob;
+  CodeBlob* _load_reference_barrier_rt_code_blob;
 
   void pre_barrier(LIRGenerator* gen, CodeEmitInfo* info, DecoratorSet decorators, LIR_Opr addr_opr, LIR_Opr pre_val);
 
@@ -185,7 +194,17 @@
   LIR_Opr ensure_in_register(LIRGenerator* gen, LIR_Opr obj);
 
 public:
-  CodeBlob* pre_barrier_c1_runtime_code_blob() { return _pre_barrier_c1_runtime_code_blob; }
+  ShenandoahBarrierSetC1();
+
+  CodeBlob* pre_barrier_c1_runtime_code_blob() {
+    assert(_pre_barrier_c1_runtime_code_blob != NULL, "");
+    return _pre_barrier_c1_runtime_code_blob;
+  }
+
+  CodeBlob* load_reference_barrier_rt_code_blob() {
+    assert(_load_reference_barrier_rt_code_blob != NULL, "");
+    return _load_reference_barrier_rt_code_blob;
+  }
 
 protected: