# HG changeset patch
# User zgu
# Date 1573051853 18000
# Node ID 83810b7d12e7ff761ad3dd91f323a22dad96f108
# Parent  a3b046720c3be73826738328f120548896e44b02
8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
Reviewed-by: aph, shade, rkennke

diff -r a3b046720c3b -r 83810b7d12e7 src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp
--- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp	Wed Nov 06 21:49:30 2019 +0900
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp	Wed Nov 06 09:50:53 2019 -0500
@@ -23,6 +23,7 @@
 
 #include "precompiled.hpp"
 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
+#include "gc/shenandoah/shenandoahConcurrentRoots.hpp"
 #include "gc/shenandoah/shenandoahForwarding.hpp"
 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
@@ -346,29 +347,44 @@
   }
 }
 
+//
+// Arguments:
+//
+// Inputs:
+//   src:        oop location to load from, might be clobbered
+//
+// Output:
+//   dst:        oop loaded from src location
+//
+// Kill:
+//   rscratch1 (scratch reg)
+//
+// Alias:
+//   dst: rscratch1 (might use rscratch1 as temporary output register to avoid clobbering src)
+//
 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
                                             Register dst, Address src, Register tmp1, Register tmp_thread) {
-  bool on_oop = is_reference_type(type);
-  bool not_in_heap = (decorators & IN_NATIVE) != 0;
-  bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
-  bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
-  bool on_reference = on_weak || on_phantom;
-  bool is_traversal_mode = ShenandoahHeap::heap()->is_traversal_mode();
-  bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0 || is_traversal_mode;
+  // 1: non-reference load, no additional barrier is needed
+  if (!is_reference_type(type)) {
+    BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
+    return;
+  }
 
-  Register result_dst = dst;
+  // 2: load a reference from src location and apply LRB if ShenandoahLoadRefBarrier is set
+  if (ShenandoahLoadRefBarrier) {
+    Register result_dst = dst;
 
-  if (on_oop) {
-    // We want to preserve src
+    // Preserve src location for LRB
     if (dst == src.base() || dst == src.index()) {
       dst = rscratch1;
     }
     assert_different_registers(dst, src.base(), src.index());
-  }
+
+    BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
 
-  BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
-  if (on_oop) {
-    if (not_in_heap && !is_traversal_mode) {
+    // Native barrier is for concurrent root processing
+    bool in_native = (decorators & IN_NATIVE) != 0;
+    if (in_native && ShenandoahConcurrentRoots::can_do_concurrent_roots()) {
       load_reference_barrier_native(masm, dst, src);
     } else {
       load_reference_barrier(masm, dst, src);
@@ -378,8 +394,19 @@
       __ mov(result_dst, dst);
       dst = result_dst;
     }
+  } else {
+    BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
+  }
 
-    if (ShenandoahKeepAliveBarrier && on_reference && keep_alive) {
+  // 3: apply keep-alive barrier if ShenandoahKeepAliveBarrier is set
+  if (ShenandoahKeepAliveBarrier) {
+    bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
+    bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
+    bool on_reference = on_weak || on_phantom;
+    bool is_traversal_mode = ShenandoahHeap::heap()->is_traversal_mode();
+    bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0 || is_traversal_mode;
+
+    if (on_reference && keep_alive) {
       __ enter();
       satb_write_barrier_pre(masm /* masm */,
                              noreg /* obj */,
diff -r a3b046720c3b -r 83810b7d12e7 src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
--- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp	Wed Nov 06 21:49:30 2019 +0900
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp	Wed Nov 06 09:50:53 2019 -0500
@@ -23,6 +23,7 @@
 
 #include "precompiled.hpp"
 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
+#include "gc/shenandoah/shenandoahConcurrentRoots.hpp"
 #include "gc/shenandoah/shenandoahForwarding.hpp"
 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
@@ -445,21 +446,33 @@
   }
 }
 
+//
+// Arguments:
+//
+// Inputs:
+//   src:        oop location, might be clobbered
+//   tmp1:       scratch register, might not be valid.
+//
+// Output:
+//   dst:        oop loaded from src location
+//
+// Kill:
+//   tmp1 (if it is valid)
+//
 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
              Register dst, Address src, Register tmp1, Register tmp_thread) {
-  bool on_oop = is_reference_type(type);
-  bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
-  bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
-  bool not_in_heap = (decorators & IN_NATIVE) != 0;
-  bool on_reference = on_weak || on_phantom;
-  bool is_traversal_mode = ShenandoahHeap::heap()->is_traversal_mode();
-  bool keep_alive = ((decorators & AS_NO_KEEPALIVE) == 0) || is_traversal_mode;
+  // 1: non-reference load, no additional barrier is needed
+  if (!is_reference_type(type)) {
+    BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
+    return;
+  }
 
-  Register result_dst = dst;
-  bool use_tmp1_for_dst = false;
+  // 2: load a reference from src location and apply LRB if ShenandoahLoadRefBarrier is set
+  if (ShenandoahLoadRefBarrier) {
+    Register result_dst = dst;
+    bool use_tmp1_for_dst = false;
 
-  if (on_oop) {
-    // We want to preserve src
+    // Preserve src location for LRB
     if (dst == src.base() || dst == src.index()) {
       // Use tmp1 for dst if possible, as it is not used in BarrierAssembler::load_at()
       if (tmp1->is_valid() && tmp1 != src.base() && tmp1 != src.index()) {
@@ -469,19 +482,20 @@
         dst = rdi;
         __ push(dst);
       }
+      assert_different_registers(dst, src.base(), src.index());
     }
-    assert_different_registers(dst, src.base(), src.index());
-  }
 
-  BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
+    BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
 
-  if (on_oop) {
-    if (not_in_heap && !is_traversal_mode) {
+    // Native barrier is for concurrent root processing
+    bool in_native = (decorators & IN_NATIVE) != 0;
+    if (in_native && ShenandoahConcurrentRoots::can_do_concurrent_roots()) {
       load_reference_barrier_native(masm, dst, src);
     } else {
       load_reference_barrier(masm, dst, src);
     }
 
+    // Move loaded oop to final destination
     if (dst != result_dst) {
       __ movptr(result_dst, dst);
 
@@ -491,8 +505,19 @@
 
       dst = result_dst;
     }
+  } else {
+    BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
+  }
 
-    if (ShenandoahKeepAliveBarrier && on_reference && keep_alive) {
+  // 3: apply keep-alive barrier if ShenandoahKeepAliveBarrier is set
+  if (ShenandoahKeepAliveBarrier) {
+    bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
+    bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
+    bool on_reference = on_weak || on_phantom;
+    bool is_traversal_mode = ShenandoahHeap::heap()->is_traversal_mode();
+    bool keep_alive = ((decorators & AS_NO_KEEPALIVE) == 0) || is_traversal_mode;
+
+    if (on_reference && keep_alive) {
       const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
       assert_different_registers(dst, tmp1, tmp_thread);
       NOT_LP64(__ get_thread(thread));
diff -r a3b046720c3b -r 83810b7d12e7 src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp
--- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp	Wed Nov 06 21:49:30 2019 +0900
+++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp	Wed Nov 06 09:50:53 2019 -0500
@@ -25,6 +25,7 @@
 #include "c1/c1_IR.hpp"
 #include "gc/shared/satbMarkQueue.hpp"
 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
+#include "gc/shenandoah/shenandoahConcurrentRoots.hpp"
 #include "gc/shenandoah/shenandoahHeap.hpp"
 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
 #include "gc/shenandoah/shenandoahRuntime.hpp"
@@ -203,46 +204,49 @@
 }
 
 void ShenandoahBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) {
+  // 1: non-reference load, no additional barrier is needed
   if (!access.is_oop()) {
     BarrierSetC1::load_at_resolved(access, result);
     return;
   }
 
   LIRGenerator* gen = access.gen();
-
   DecoratorSet decorators = access.decorators();
-  bool is_traversal_mode = ShenandoahHeap::heap()->is_traversal_mode();
 
-  if ((decorators & IN_NATIVE) != 0 && !is_traversal_mode) {
-    assert(access.is_oop(), "IN_NATIVE access only for oop values");
-    BarrierSetC1::load_at_resolved(access, result);
-    LIR_OprList* args = new LIR_OprList();
-    LIR_Opr addr = access.resolved_addr();
-    addr = ensure_in_register(gen, addr);
-    args->append(result);
-    args->append(addr);
-    BasicTypeList signature;
-    signature.append(T_OBJECT);
-    signature.append(T_ADDRESS);
-    LIR_Opr call_result = gen->call_runtime(&signature, args,
-                                            CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native),
-                                            objectType, NULL);
-    __ move(call_result, result);
-  } else {
-    if (ShenandoahLoadRefBarrier) {
+  // 2: load a reference from src location and apply LRB if ShenandoahLoadRefBarrier is set
+  if (ShenandoahLoadRefBarrier) {
+    // Native barrier is for concurrent root processing
+    bool in_native = (decorators & IN_NATIVE) != 0;
+    if (in_native && ShenandoahConcurrentRoots::can_do_concurrent_roots()) {
+      BarrierSetC1::load_at_resolved(access, result);
+      LIR_OprList* args = new LIR_OprList();
+      LIR_Opr addr = access.resolved_addr();
+      addr = ensure_in_register(gen, addr);
+      args->append(result);
+      args->append(addr);
+      BasicTypeList signature;
+      signature.append(T_OBJECT);
+      signature.append(T_ADDRESS);
+      LIR_Opr call_result = gen->call_runtime(&signature, args,
+                                              CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native),
+                                              objectType, NULL);
+      __ move(call_result, result);
+    } else {
       LIR_Opr tmp = gen->new_register(T_OBJECT);
       BarrierSetC1::load_at_resolved(access, tmp);
-      tmp = load_reference_barrier(access.gen(), tmp, access.resolved_addr());
+      tmp = load_reference_barrier(gen, tmp, access.resolved_addr());
       __ move(tmp, result);
-    } else {
-      BarrierSetC1::load_at_resolved(access, result);
     }
+  } else {
+    BarrierSetC1::load_at_resolved(access, result);
   }
 
+  // 3: apply keep-alive barrier if ShenandoahKeepAliveBarrier is set
   if (ShenandoahKeepAliveBarrier) {
     bool is_weak = (decorators & ON_WEAK_OOP_REF) != 0;
     bool is_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
     bool is_anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
+    bool is_traversal_mode = ShenandoahHeap::heap()->is_traversal_mode();
     bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0 || is_traversal_mode;
 
     if ((is_weak || is_phantom || is_anonymous) && keep_alive) {
@@ -252,13 +256,13 @@
         Lcont_anonymous = new LabelObj();
         generate_referent_check(access, Lcont_anonymous);
       }
-      pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr /* addr_opr */,
+      pre_barrier(gen, access.access_emit_info(), decorators, LIR_OprFact::illegalOpr /* addr_opr */,
                   result /* pre_val */);
       if (is_anonymous) {
         __ branch_destination(Lcont_anonymous->label());
       }
     }
-  }
+ }
 }
 
 class C1ShenandoahPreBarrierCodeGenClosure : public StubAssemblerCodeGenClosure {
diff -r a3b046720c3b -r 83810b7d12e7 src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp	Wed Nov 06 21:49:30 2019 +0900
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp	Wed Nov 06 09:50:53 2019 -0500
@@ -23,6 +23,7 @@
 
 #include "precompiled.hpp"
 #include "gc/shared/barrierSet.hpp"
+#include "gc/shenandoah/shenandoahConcurrentRoots.hpp"
 #include "gc/shenandoah/shenandoahForwarding.hpp"
 #include "gc/shenandoah/shenandoahHeap.hpp"
 #include "gc/shenandoah/shenandoahHeuristics.hpp"
@@ -534,66 +535,69 @@
 }
 
 Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
-  DecoratorSet decorators = access.decorators();
-
-  Node* adr = access.addr().node();
-  Node* obj = access.base();
+  // 1: non-reference load, no additional barrier is needed
+  if (!access.is_oop()) {
+    return BarrierSetC2::load_at_resolved(access, val_type);;
+  }
 
-  bool mismatched = (decorators & C2_MISMATCHED) != 0;
-  bool unknown = (decorators & ON_UNKNOWN_OOP_REF) != 0;
-  bool on_heap = (decorators & IN_HEAP) != 0;
-  bool on_weak_ref = (decorators & (ON_WEAK_OOP_REF | ON_PHANTOM_OOP_REF)) != 0;
-  bool is_unordered = (decorators & MO_UNORDERED) != 0;
-  bool need_cpu_mem_bar = !is_unordered || mismatched || !on_heap;
-  bool is_traversal_mode = ShenandoahHeap::heap()->is_traversal_mode();
-  bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0 || is_traversal_mode;
+  Node* load = BarrierSetC2::load_at_resolved(access, val_type);
+  DecoratorSet decorators = access.decorators();
   bool in_native = (decorators & IN_NATIVE) != 0;
 
-  Node* top = Compile::current()->top();
-
-  Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : top;
-  Node* load = BarrierSetC2::load_at_resolved(access, val_type);
-
-  if (access.is_oop()) {
-    if (ShenandoahLoadRefBarrier) {
-      load = new ShenandoahLoadReferenceBarrierNode(NULL, load, in_native && !is_traversal_mode);
-      if (access.is_parse_access()) {
-        load = static_cast<C2ParseAccess &>(access).kit()->gvn().transform(load);
-      } else {
-        load = static_cast<C2OptAccess &>(access).gvn().transform(load);
-      }
+  // 2: apply LRB if ShenandoahLoadRefBarrier is set
+  if (ShenandoahLoadRefBarrier) {
+    // Native barrier is for concurrent root processing
+    bool use_native_barrier = in_native && ShenandoahConcurrentRoots::can_do_concurrent_roots();
+    load = new ShenandoahLoadReferenceBarrierNode(NULL, load, use_native_barrier);
+    if (access.is_parse_access()) {
+      load = static_cast<C2ParseAccess &>(access).kit()->gvn().transform(load);
+    } else {
+      load = static_cast<C2OptAccess &>(access).gvn().transform(load);
     }
   }
 
-  // If we are reading the value of the referent field of a Reference
-  // object (either by using Unsafe directly or through reflection)
-  // then, if SATB is enabled, we need to record the referent in an
-  // SATB log buffer using the pre-barrier mechanism.
-  // Also we need to add memory barrier to prevent commoning reads
-  // from this field across safepoint since GC can change its value.
-  bool need_read_barrier = ShenandoahKeepAliveBarrier &&
-    (on_weak_ref || (unknown && offset != top && obj != top));
+  // 3: apply keep-alive barrier if ShenandoahKeepAliveBarrier is set
+  if (ShenandoahKeepAliveBarrier) {
+    Node* top = Compile::current()->top();
+    Node* adr = access.addr().node();
+    Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : top;
+    Node* obj = access.base();
 
-  if (!access.is_oop() || !need_read_barrier) {
-    return load;
-  }
+    bool unknown = (decorators & ON_UNKNOWN_OOP_REF) != 0;
+    bool on_weak_ref = (decorators & (ON_WEAK_OOP_REF | ON_PHANTOM_OOP_REF)) != 0;
+    bool is_traversal_mode = ShenandoahHeap::heap()->is_traversal_mode();
+    bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0 || is_traversal_mode;
 
-  assert(access.is_parse_access(), "entry not supported at optimization time");
-  C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
-  GraphKit* kit = parse_access.kit();
+    // If we are reading the value of the referent field of a Reference
+    // object (either by using Unsafe directly or through reflection)
+    // then, if SATB is enabled, we need to record the referent in an
+    // SATB log buffer using the pre-barrier mechanism.
+    // Also we need to add memory barrier to prevent commoning reads
+    // from this field across safepoint since GC can change its value.
+    if (!on_weak_ref || (unknown && (offset == top || obj == top)) || !keep_alive) {
+      return load;
+    }
 
-  if (on_weak_ref && keep_alive) {
-    // Use the pre-barrier to record the value in the referent field
-    satb_write_barrier_pre(kit, false /* do_load */,
-                           NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
-                           load /* pre_val */, T_OBJECT);
-    // Add memory barrier to prevent commoning reads from this field
-    // across safepoint since GC can change its value.
-    kit->insert_mem_bar(Op_MemBarCPUOrder);
-  } else if (unknown) {
-    // We do not require a mem bar inside pre_barrier if need_mem_bar
-    // is set: the barriers would be emitted by us.
-    insert_pre_barrier(kit, obj, offset, load, !need_cpu_mem_bar);
+    assert(access.is_parse_access(), "entry not supported at optimization time");
+    C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
+    GraphKit* kit = parse_access.kit();
+    bool mismatched = (decorators & C2_MISMATCHED) != 0;
+    bool is_unordered = (decorators & MO_UNORDERED) != 0;
+    bool need_cpu_mem_bar = !is_unordered || mismatched || in_native;
+
+    if (on_weak_ref) {
+      // Use the pre-barrier to record the value in the referent field
+      satb_write_barrier_pre(kit, false /* do_load */,
+                             NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
+                             load /* pre_val */, T_OBJECT);
+      // Add memory barrier to prevent commoning reads from this field
+      // across safepoint since GC can change its value.
+      kit->insert_mem_bar(Op_MemBarCPUOrder);
+    } else if (unknown) {
+      // We do not require a mem bar inside pre_barrier if need_mem_bar
+      // is set: the barriers would be emitted by us.
+      insert_pre_barrier(kit, obj, offset, load, !need_cpu_mem_bar);
+    }
   }
 
   return load;