8232896: ZGC: Enable C2 clone intrinsic
authorneliasso
Thu, 31 Oct 2019 17:16:36 +0100
changeset 58931 304c63b17b07
parent 58930 a4ddd1667c72
child 58932 8623f75be895
8232896: ZGC: Enable C2 clone intrinsic Reviewed-by: pliden, kvn
src/hotspot/share/classfile/vmSymbols.cpp
src/hotspot/share/gc/shared/c2/barrierSetC2.cpp
src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp
src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp
src/hotspot/share/gc/z/zBarrier.hpp
src/hotspot/share/gc/z/zBarrier.inline.hpp
src/hotspot/share/gc/z/zBarrierSetRuntime.cpp
src/hotspot/share/gc/z/zBarrierSetRuntime.hpp
src/hotspot/share/opto/arraycopynode.hpp
src/hotspot/share/opto/library_call.cpp
src/hotspot/share/opto/macroArrayCopy.cpp
src/hotspot/share/opto/type.hpp
test/micro/org/openjdk/bench/java/lang/Clone.java
--- a/src/hotspot/share/classfile/vmSymbols.cpp	Tue Nov 05 10:11:18 2019 +0000
+++ b/src/hotspot/share/classfile/vmSymbols.cpp	Thu Oct 31 17:16:36 2019 +0100
@@ -786,9 +786,6 @@
 #endif // COMPILER1
 #ifdef COMPILER2
   case vmIntrinsics::_clone:
-#if INCLUDE_ZGC
-    if (UseZGC) return true;
-#endif
   case vmIntrinsics::_copyOf:
   case vmIntrinsics::_copyOfRange:
     // These intrinsics use both the objectcopy and the arraycopy
--- a/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp	Tue Nov 05 10:11:18 2019 +0000
+++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp	Thu Oct 31 17:16:36 2019 +0100
@@ -96,7 +96,6 @@
 
     store = kit->store_to_memory(kit->control(), access.addr().node(), val.node(), access.type(),
                                      access.addr().type(), mo, requires_atomic_access, unaligned, mismatched, unsafe);
-    access.set_raw_access(store);
   } else {
     assert(!requires_atomic_access, "not yet supported");
     assert(access.is_opt_access(), "either parse or opt access");
@@ -120,6 +119,8 @@
       mm->set_memory_at(alias, st);
     }
   }
+  access.set_raw_access(store);
+
   return store;
 }
 
@@ -153,7 +154,6 @@
       load = kit->make_load(control, adr, val_type, access.type(), adr_type, mo,
                             dep, requires_atomic_access, unaligned, mismatched, unsafe);
     }
-    access.set_raw_access(load);
   } else {
     assert(!requires_atomic_access, "not yet supported");
     assert(access.is_opt_access(), "either parse or opt access");
@@ -165,6 +165,7 @@
     load = LoadNode::make(gvn, control, mem, adr, adr_type, val_type, access.type(), mo, dep, unaligned, mismatched);
     load = gvn.transform(load);
   }
+  access.set_raw_access(load);
 
   return load;
 }
@@ -806,7 +807,8 @@
   Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
   Node* length = ac->in(ArrayCopyNode::Length);
 
-  assert (src_offset == NULL && dest_offset == NULL, "for clone offsets should be null");
+  assert (src_offset == NULL,  "for clone offsets should be null");
+  assert (dest_offset == NULL, "for clone offsets should be null");
 
   const char* copyfunc_name = "arraycopy";
   address     copyfunc_addr =
--- a/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp	Tue Nov 05 10:11:18 2019 +0000
+++ b/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp	Thu Oct 31 17:16:36 2019 +0100
@@ -27,14 +27,17 @@
 #include "gc/z/zBarrierSet.hpp"
 #include "gc/z/zBarrierSetAssembler.hpp"
 #include "gc/z/zBarrierSetRuntime.hpp"
+#include "opto/arraycopynode.hpp"
 #include "opto/block.hpp"
 #include "opto/compile.hpp"
 #include "opto/graphKit.hpp"
 #include "opto/machnode.hpp"
+#include "opto/macro.hpp"
 #include "opto/memnode.hpp"
 #include "opto/node.hpp"
 #include "opto/regalloc.hpp"
 #include "opto/rootnode.hpp"
+#include "opto/runtime.hpp"
 #include "utilities/growableArray.hpp"
 #include "utilities/macros.hpp"
 
@@ -429,3 +432,50 @@
     }
   }
 }
+
+const TypeFunc *oop_clone_Type() {
+  // create input type (domain)
+  const Type **fields = TypeTuple::fields(3);
+  fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL;  // src Object
+  fields[TypeFunc::Parms+1] = TypeInstPtr::NOTNULL;  // dst Object
+  fields[TypeFunc::Parms+2] = TypeInt::INT;          // Object size
+  const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+3, fields);
+
+  // create result type (range)
+  fields = TypeTuple::fields(0);
+
+  const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
+
+  return TypeFunc::make(domain, range);
+}
+
+void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
+  Node *ctrl = ac->in(TypeFunc::Control);
+  Node *mem = ac->in(TypeFunc::Memory);
+  Node *src = ac->in(ArrayCopyNode::Src);
+  Node *src_offset = ac->in(ArrayCopyNode::SrcPos);
+  Node *dest = ac->in(ArrayCopyNode::Dest);
+  Node *dest_offset = ac->in(ArrayCopyNode::DestPos);
+  Node *length = ac->in(ArrayCopyNode::Length);
+
+  assert (src_offset == NULL,  "for clone offsets should be null");
+  assert (dest_offset == NULL, "for clone offsets should be null");
+
+  if (src->bottom_type()->isa_instptr()) {
+    // Instances must have all oop fiels healed before cloning - call runtime leaf
+    const char *clonefunc_name = "clone_oop";
+    address clonefunc_addr = ZBarrierSetRuntime::clone_oop_addr();
+    const TypePtr *raw_adr_type = TypeRawPtr::BOTTOM;
+    const TypeFunc *call_type = oop_clone_Type();
+
+    Node *call = phase->make_leaf_call(ctrl, mem, call_type, clonefunc_addr, clonefunc_name, raw_adr_type, src, dest,
+                                       length);
+    phase->transform_later(call);
+    phase->igvn().replace_node(ac, call);
+  } else {
+    assert(src->bottom_type()->isa_aryptr() != NULL, "Only arrays");
+
+    // Clones of primitive arrays go here
+    BarrierSetC2::clone_at_expansion(phase, ac);
+  }
+}
--- a/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp	Tue Nov 05 10:11:18 2019 +0000
+++ b/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp	Thu Oct 31 17:16:36 2019 +0100
@@ -86,6 +86,7 @@
   virtual void late_barrier_analysis() const;
   virtual int estimate_stub_size() const;
   virtual void emit_stubs(CodeBuffer& cb) const;
+  virtual void clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const;
 };
 
 #endif // SHARE_GC_Z_C2_ZBARRIERSETC2_HPP
--- a/src/hotspot/share/gc/z/zBarrier.hpp	Tue Nov 05 10:11:18 2019 +0000
+++ b/src/hotspot/share/gc/z/zBarrier.hpp	Thu Oct 31 17:16:36 2019 +0100
@@ -119,6 +119,7 @@
   static oop  load_barrier_on_oop_field(volatile narrowOop* p);
   static oop  load_barrier_on_oop_field_preloaded(volatile narrowOop* p, oop o);
   static void load_barrier_on_oop_array(volatile narrowOop* p, size_t length);
+  static void clone_oop(volatile oop src, oop dst, size_t length);
   static oop  load_barrier_on_weak_oop_field_preloaded(volatile narrowOop* p, oop o);
   static oop  load_barrier_on_phantom_oop_field_preloaded(volatile narrowOop* p, oop o);
   static oop  weak_load_barrier_on_oop_field_preloaded(volatile narrowOop* p, oop o);
--- a/src/hotspot/share/gc/z/zBarrier.inline.hpp	Tue Nov 05 10:11:18 2019 +0000
+++ b/src/hotspot/share/gc/z/zBarrier.inline.hpp	Thu Oct 31 17:16:36 2019 +0100
@@ -175,6 +175,10 @@
   }
 }
 
+inline void ZBarrier::clone_oop(volatile oop src, oop dst, size_t length) {
+  HeapAccess<>::clone(src, dst, length);
+}
+
 // ON_WEAK barriers should only ever be applied to j.l.r.Reference.referents.
 inline void verify_on_weak(volatile oop* referent_addr) {
 #ifdef ASSERT
--- a/src/hotspot/share/gc/z/zBarrierSetRuntime.cpp	Tue Nov 05 10:11:18 2019 +0000
+++ b/src/hotspot/share/gc/z/zBarrierSetRuntime.cpp	Thu Oct 31 17:16:36 2019 +0100
@@ -42,6 +42,10 @@
   ZBarrier::load_barrier_on_oop_array(p, length);
 JRT_END
 
+JRT_LEAF(void, ZBarrierSetRuntime::clone_oop(oop src, oop dst, size_t length))
+  ZBarrier::clone_oop(src, dst, length);
+JRT_END
+
 address ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(DecoratorSet decorators) {
   if (decorators & ON_PHANTOM_OOP_REF) {
     return load_barrier_on_phantom_oop_field_preloaded_addr();
@@ -67,3 +71,7 @@
 address ZBarrierSetRuntime::load_barrier_on_oop_array_addr() {
   return reinterpret_cast<address>(load_barrier_on_oop_array);
 }
+
+address ZBarrierSetRuntime::clone_oop_addr() {
+  return reinterpret_cast<address>(clone_oop);
+}
--- a/src/hotspot/share/gc/z/zBarrierSetRuntime.hpp	Tue Nov 05 10:11:18 2019 +0000
+++ b/src/hotspot/share/gc/z/zBarrierSetRuntime.hpp	Thu Oct 31 17:16:36 2019 +0100
@@ -36,6 +36,7 @@
   static oopDesc* load_barrier_on_weak_oop_field_preloaded(oopDesc* o, oop* p);
   static oopDesc* load_barrier_on_phantom_oop_field_preloaded(oopDesc* o, oop* p);
   static void load_barrier_on_oop_array(oop* p, size_t length);
+  static void clone_oop(oop src, oop dst, size_t length);
 
 public:
   static address load_barrier_on_oop_field_preloaded_addr(DecoratorSet decorators);
@@ -43,6 +44,7 @@
   static address load_barrier_on_weak_oop_field_preloaded_addr();
   static address load_barrier_on_phantom_oop_field_preloaded_addr();
   static address load_barrier_on_oop_array_addr();
+  static address clone_oop_addr();
 };
 
 #endif // SHARE_GC_Z_ZBARRIERSETRUNTIME_HPP
--- a/src/hotspot/share/opto/arraycopynode.hpp	Tue Nov 05 10:11:18 2019 +0000
+++ b/src/hotspot/share/opto/arraycopynode.hpp	Thu Oct 31 17:16:36 2019 +0100
@@ -38,7 +38,7 @@
     None,            // not set yet
     ArrayCopy,       // System.arraycopy()
     CloneBasic,      // A clone that can be copied by 64 bit chunks
-    CloneOop,        // An oop array clone
+    CloneOopArray,   // An oop array clone
     CopyOf,          // Arrays.copyOf()
     CopyOfRange      // Arrays.copyOfRange()
   } _kind;
@@ -147,7 +147,7 @@
   bool is_arraycopy()             const  { assert(_kind != None, "should bet set"); return _kind == ArrayCopy; }
   bool is_arraycopy_validated()   const  { assert(_kind != None, "should bet set"); return _kind == ArrayCopy && _arguments_validated; }
   bool is_clonebasic()            const  { assert(_kind != None, "should bet set"); return _kind == CloneBasic; }
-  bool is_cloneoop()              const  { assert(_kind != None, "should bet set"); return _kind == CloneOop; }
+  bool is_clone_oop_array()       const  { assert(_kind != None, "should bet set"); return _kind == CloneOopArray; }
   bool is_copyof()                const  { assert(_kind != None, "should bet set"); return _kind == CopyOf; }
   bool is_copyof_validated()      const  { assert(_kind != None, "should bet set"); return _kind == CopyOf && _arguments_validated; }
   bool is_copyofrange()           const  { assert(_kind != None, "should bet set"); return _kind == CopyOfRange; }
@@ -155,7 +155,7 @@
 
   void set_arraycopy(bool validated)   { assert(_kind == None, "shouldn't bet set yet"); _kind = ArrayCopy; _arguments_validated = validated; }
   void set_clonebasic()                { assert(_kind == None, "shouldn't bet set yet"); _kind = CloneBasic; }
-  void set_cloneoop()                  { assert(_kind == None, "shouldn't bet set yet"); _kind = CloneOop; }
+  void set_clone_oop_array()           { assert(_kind == None, "shouldn't bet set yet"); _kind = CloneOopArray; }
   void set_copyof(bool validated)      { assert(_kind == None, "shouldn't bet set yet"); _kind = CopyOf; _arguments_validated = validated; }
   void set_copyofrange(bool validated) { assert(_kind == None, "shouldn't bet set yet"); _kind = CopyOfRange; _arguments_validated = validated; }
 
--- a/src/hotspot/share/opto/library_call.cpp	Tue Nov 05 10:11:18 2019 +0000
+++ b/src/hotspot/share/opto/library_call.cpp	Thu Oct 31 17:16:36 2019 +0100
@@ -4234,10 +4234,7 @@
     alloc->initialization()->set_complete_with_arraycopy();
   }
 
-  // Copy the fastest available way.
-  // TODO: generate fields copies for small objects instead.
   Node* size = _gvn.transform(obj_size);
-
   access_clone(obj, alloc_obj, size, is_array);
 
   // Do not let reads from the cloned object float above the arraycopy.
@@ -4304,12 +4301,6 @@
       }
     }
 
-    Node* obj_klass = load_object_klass(obj);
-    const TypeKlassPtr* tklass = _gvn.type(obj_klass)->isa_klassptr();
-    const TypeOopPtr*   toop   = ((tklass != NULL)
-                                ? tklass->as_instance_type()
-                                : TypeInstPtr::NOTNULL);
-
     // Conservatively insert a memory barrier on all memory slices.
     // Do not let writes into the original float below the clone.
     insert_mem_bar(Op_MemBarCPUOrder);
@@ -4328,6 +4319,7 @@
     PhiNode*    result_mem = new PhiNode(result_reg, Type::MEMORY, TypePtr::BOTTOM);
     record_for_igvn(result_reg);
 
+    Node* obj_klass = load_object_klass(obj);
     Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL);
     if (array_ctl != NULL) {
       // It's an array.
@@ -4349,7 +4341,7 @@
           // Generate a direct call to the right arraycopy function(s).
           Node* alloc = tightly_coupled_allocation(alloc_obj, NULL);
           ArrayCopyNode* ac = ArrayCopyNode::make(this, true, obj, intcon(0), alloc_obj, intcon(0), obj_length, alloc != NULL, false);
-          ac->set_cloneoop();
+          ac->set_clone_oop_array();
           Node* n = _gvn.transform(ac);
           assert(n == ac, "cannot disappear");
           ac->connect_outputs(this);
--- a/src/hotspot/share/opto/macroArrayCopy.cpp	Tue Nov 05 10:11:18 2019 +0000
+++ b/src/hotspot/share/opto/macroArrayCopy.cpp	Thu Oct 31 17:16:36 2019 +0100
@@ -505,7 +505,7 @@
 
     // We don't need a subtype check for validated copies and Object[].clone()
     bool skip_subtype_check = ac->is_arraycopy_validated() || ac->is_copyof_validated() ||
-                              ac->is_copyofrange_validated() || ac->is_cloneoop();
+                              ac->is_copyofrange_validated() || ac->is_clone_oop_array();
     if (!skip_subtype_check) {
       // Get the klass* for both src and dest
       Node* src_klass  = ac->in(ArrayCopyNode::SrcKlass);
@@ -1096,7 +1096,7 @@
     BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
     bs->clone_at_expansion(this, ac);
     return;
-  } else if (ac->is_copyof() || ac->is_copyofrange() || ac->is_cloneoop()) {
+  } else if (ac->is_copyof() || ac->is_copyofrange() || ac->is_clone_oop_array()) {
     Node* mem = ac->in(TypeFunc::Memory);
     merge_mem = MergeMemNode::make(mem);
     transform_later(merge_mem);
--- a/src/hotspot/share/opto/type.hpp	Tue Nov 05 10:11:18 2019 +0000
+++ b/src/hotspot/share/opto/type.hpp	Thu Oct 31 17:16:36 2019 +0100
@@ -290,6 +290,7 @@
   const TypeF      *isa_float_constant() const;  // Returns NULL if not a FloatCon
   const TypeTuple  *is_tuple() const;            // Collection of fields, NOT a pointer
   const TypeAry    *is_ary() const;              // Array, NOT array pointer
+  const TypeAry    *isa_ary() const;             // Returns NULL of not ary
   const TypeVect   *is_vect() const;             // Vector
   const TypeVect   *isa_vect() const;            // Returns NULL if not a Vector
   const TypePtr    *is_ptr() const;              // Asserts it is a ptr type
@@ -1615,6 +1616,10 @@
   return (TypeAry*)this;
 }
 
+inline const TypeAry *Type::isa_ary() const {
+  return ((_base == Array) ? (TypeAry*)this : NULL);
+}
+
 inline const TypeVect *Type::is_vect() const {
   assert( _base >= VectorS && _base <= VectorZ, "Not a Vector" );
   return (TypeVect*)this;
--- a/test/micro/org/openjdk/bench/java/lang/Clone.java	Tue Nov 05 10:11:18 2019 +0000
+++ b/test/micro/org/openjdk/bench/java/lang/Clone.java	Thu Oct 31 17:16:36 2019 +0100
@@ -43,12 +43,17 @@
     private BitSet testObj1;
     private Date testObj2;
     private char[] testObj3;
+    private char[] testObj4;
+    private String[] testObj5;
 
     @Setup
     public void setup() {
         testObj1 = new BitSet(10);
         testObj2 = new Date();
         testObj3 = new char[5];
+        testObj4 = new char[311];
+        String str = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut";
+        testObj5 = str.split(" ", -1);
     }
 
     /** Calls clone on three different types. The types are java.util.BitSet, java.util.Date and char[]. */
@@ -59,5 +64,11 @@
         bh.consume(testObj3.clone());
     }
 
+    @Benchmark
+    public void cloneLarge(Blackhole bh) {
+        bh.consume(testObj4.clone());
+        bh.consume(testObj5.clone());
+    }
+
 }