7173584: Implement arraycopy as a macro node
Summary: delay the conversion of arraycopy to stub calls to macro expansion
Reviewed-by: kvn, iveresov
--- a/hotspot/src/share/vm/opto/callnode.cpp Fri Aug 08 11:36:48 2014 -0700
+++ b/hotspot/src/share/vm/opto/callnode.cpp Mon Aug 11 14:12:51 2014 +0200
@@ -1815,3 +1815,49 @@
}
return result;
}
+
+ArrayCopyNode::ArrayCopyNode(Compile* C, bool alloc_tightly_coupled)
+ : CallNode(arraycopy_type(), NULL, TypeRawPtr::BOTTOM), _alloc_tightly_coupled(alloc_tightly_coupled), _kind(ArrayCopy) {
+ init_class_id(Class_ArrayCopy);
+ init_flags(Flag_is_macro);
+ C->add_macro_node(this);
+}
+
+uint ArrayCopyNode::size_of() const { return sizeof(*this); }
+
+ArrayCopyNode* ArrayCopyNode::make(GraphKit* kit, bool may_throw,
+ Node* src, Node* src_offset, Node* dest, Node* dest_offset, Node* length,
+ bool alloc_tightly_coupled) {
+
+ ArrayCopyNode* ac = new ArrayCopyNode(kit->C, alloc_tightly_coupled);
+ Node* prev_mem = kit->set_predefined_input_for_runtime_call(ac);
+
+ ac->init_req( ArrayCopyNode::Src, src);
+ ac->init_req( ArrayCopyNode::SrcPos, src_offset);
+ ac->init_req( ArrayCopyNode::Dest, dest);
+ ac->init_req( ArrayCopyNode::DestPos, dest_offset);
+ ac->init_req( ArrayCopyNode::Length, length);
+
+ if (may_throw) {
+ ac->set_req(TypeFunc::I_O , kit->i_o());
+ kit->add_safepoint_edges(ac, false);
+ }
+
+ return ac;
+}
+
+void ArrayCopyNode::connect_outputs(GraphKit* kit) {
+ kit->set_all_memory_call(this, true);
+ kit->set_control(kit->gvn().transform(new ProjNode(this,TypeFunc::Control)));
+ kit->set_i_o(kit->gvn().transform(new ProjNode(this, TypeFunc::I_O)));
+ kit->make_slow_call_ex(this, kit->env()->Throwable_klass(), true);
+ kit->set_all_memory_call(this);
+}
+
+#ifndef PRODUCT
+const char* ArrayCopyNode::_kind_names[] = {"arraycopy", "arraycopy, validated arguments", "clone", "oop array clone", "CopyOf", "CopyOfRange"};
+void ArrayCopyNode::dump_spec(outputStream *st) const {
+ CallNode::dump_spec(st);
+ st->print(" (%s%s)", _kind_names[_kind], _alloc_tightly_coupled ? ", tightly coupled allocation" : "");
+}
+#endif
--- a/hotspot/src/share/vm/opto/callnode.hpp Fri Aug 08 11:36:48 2014 -0700
+++ b/hotspot/src/share/vm/opto/callnode.hpp Mon Aug 11 14:12:51 2014 +0200
@@ -1063,4 +1063,96 @@
virtual bool guaranteed_safepoint() { return false; }
};
+class GraphKit;
+
+class ArrayCopyNode : public CallNode {
+private:
+
+ // What kind of arraycopy variant is this?
+ enum {
+ ArrayCopy, // System.arraycopy()
+ ArrayCopyNoTest, // System.arraycopy(), all arguments validated
+ CloneBasic, // A clone that can be copied by 64 bit chunks
+ CloneOop, // An oop array clone
+ CopyOf, // Arrays.copyOf()
+ CopyOfRange // Arrays.copyOfRange()
+ } _kind;
+
+#ifndef PRODUCT
+ static const char* _kind_names[CopyOfRange+1];
+#endif
+ // Is the alloc obtained with
+ // AllocateArrayNode::Ideal_array_allocation() tighly coupled
+ // (arraycopy follows immediately the allocation)?
+ // We cache the result of LibraryCallKit::tightly_coupled_allocation
+ // here because it's much easier to find whether there's a tightly
+ // couple allocation at parse time than at macro expansion time. At
+ // macro expansion time, for every use of the allocation node we
+ // would need to figure out whether it happens after the arraycopy (and
+ // can be ignored) or between the allocation and the arraycopy. At
+ // parse time, it's straightforward because whatever happens after
+ // the arraycopy is not parsed yet so doesn't exist when
+ // LibraryCallKit::tightly_coupled_allocation() is called.
+ bool _alloc_tightly_coupled;
+
+ static const TypeFunc* arraycopy_type() {
+ const Type** fields = TypeTuple::fields(ParmLimit - TypeFunc::Parms);
+ fields[Src] = TypeInstPtr::BOTTOM;
+ fields[SrcPos] = TypeInt::INT;
+ fields[Dest] = TypeInstPtr::BOTTOM;
+ fields[DestPos] = TypeInt::INT;
+ fields[Length] = TypeInt::INT;
+ const TypeTuple *domain = TypeTuple::make(ParmLimit, fields);
+
+ // create result type (range)
+ fields = TypeTuple::fields(0);
+
+ const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
+
+ return TypeFunc::make(domain, range);
+ }
+
+ ArrayCopyNode(Compile* C, bool alloc_tightly_coupled);
+
+public:
+
+ enum {
+ Src = TypeFunc::Parms,
+ SrcPos,
+ Dest,
+ DestPos,
+ Length,
+ ParmLimit
+ };
+
+ static ArrayCopyNode* make(GraphKit* kit, bool may_throw,
+ Node* src, Node* src_offset, Node* dest, Node* dest_offset, Node* length,
+ bool alloc_tightly_coupled);
+
+ void connect_outputs(GraphKit* kit);
+
+ bool is_arraycopy() const { return _kind == ArrayCopy; }
+ bool is_arraycopy_notest() const { return _kind == ArrayCopyNoTest; }
+ bool is_clonebasic() const { return _kind == CloneBasic; }
+ bool is_cloneoop() const { return _kind == CloneOop; }
+ bool is_copyof() const { return _kind == CopyOf; }
+ bool is_copyofrange() const { return _kind == CopyOfRange; }
+
+ void set_arraycopy() { _kind = ArrayCopy; }
+ void set_arraycopy_notest() { _kind = ArrayCopyNoTest; }
+ void set_clonebasic() { _kind = CloneBasic; }
+ void set_cloneoop() { _kind = CloneOop; }
+ void set_copyof() { _kind = CopyOf; }
+ void set_copyofrange() { _kind = CopyOfRange; }
+
+ virtual int Opcode() const;
+ virtual uint size_of() const; // Size is bigger
+ virtual bool guaranteed_safepoint() { return false; }
+
+ bool is_alloc_tightly_coupled() const { return _alloc_tightly_coupled; }
+
+#ifndef PRODUCT
+ virtual void dump_spec(outputStream *st) const;
+#endif
+};
#endif // SHARE_VM_OPTO_CALLNODE_HPP
--- a/hotspot/src/share/vm/opto/classes.hpp Fri Aug 08 11:36:48 2014 -0700
+++ b/hotspot/src/share/vm/opto/classes.hpp Mon Aug 11 14:12:51 2014 +0200
@@ -37,6 +37,7 @@
macro(AllocateArray)
macro(AndI)
macro(AndL)
+macro(ArrayCopy)
macro(AryEq)
macro(AtanD)
macro(Binary)
--- a/hotspot/src/share/vm/opto/compile.cpp Fri Aug 08 11:36:48 2014 -0700
+++ b/hotspot/src/share/vm/opto/compile.cpp Mon Aug 11 14:12:51 2014 +0200
@@ -3787,6 +3787,56 @@
}
}
+//----------------------------static_subtype_check-----------------------------
+// Shortcut important common cases when superklass is exact:
+// (0) superklass is java.lang.Object (can occur in reflective code)
+// (1) subklass is already limited to a subtype of superklass => always ok
+// (2) subklass does not overlap with superklass => always fail
+// (3) superklass has NO subtypes and we can check with a simple compare.
+int Compile::static_subtype_check(ciKlass* superk, ciKlass* subk) {
+ if (StressReflectiveCode) {
+ return SSC_full_test; // Let caller generate the general case.
+ }
+
+ if (superk == env()->Object_klass()) {
+ return SSC_always_true; // (0) this test cannot fail
+ }
+
+ ciType* superelem = superk;
+ if (superelem->is_array_klass())
+ superelem = superelem->as_array_klass()->base_element_type();
+
+ if (!subk->is_interface()) { // cannot trust static interface types yet
+ if (subk->is_subtype_of(superk)) {
+ return SSC_always_true; // (1) false path dead; no dynamic test needed
+ }
+ if (!(superelem->is_klass() && superelem->as_klass()->is_interface()) &&
+ !superk->is_subtype_of(subk)) {
+ return SSC_always_false;
+ }
+ }
+
+ // If casting to an instance klass, it must have no subtypes
+ if (superk->is_interface()) {
+ // Cannot trust interfaces yet.
+ // %%% S.B. superk->nof_implementors() == 1
+ } else if (superelem->is_instance_klass()) {
+ ciInstanceKlass* ik = superelem->as_instance_klass();
+ if (!ik->has_subklass() && !ik->is_interface()) {
+ if (!ik->is_final()) {
+ // Add a dependency if there is a chance of a later subclass.
+ dependencies()->assert_leaf_type(ik);
+ }
+ return SSC_easy_test; // (3) caller can do a simple ptr comparison
+ }
+ } else {
+ // A primitive array type has no subtypes.
+ return SSC_easy_test; // (3) caller can do a simple ptr comparison
+ }
+
+ return SSC_full_test;
+}
+
// The message about the current inlining is accumulated in
// _print_inlining_stream and transfered into the _print_inlining_list
// once we know whether inlining succeeds or not. For regular
--- a/hotspot/src/share/vm/opto/compile.hpp Fri Aug 08 11:36:48 2014 -0700
+++ b/hotspot/src/share/vm/opto/compile.hpp Mon Aug 11 14:12:51 2014 +0200
@@ -1200,6 +1200,10 @@
// Definitions of pd methods
static void pd_compiler2_init();
+ // Static parse-time type checking logic for gen_subtype_check:
+ enum { SSC_always_false, SSC_always_true, SSC_easy_test, SSC_full_test };
+ int static_subtype_check(ciKlass* superk, ciKlass* subk);
+
// Auxiliary method for randomized fuzzing/stressing
static bool randomized_select(int count);
};
--- a/hotspot/src/share/vm/opto/graphKit.cpp Fri Aug 08 11:36:48 2014 -0700
+++ b/hotspot/src/share/vm/opto/graphKit.cpp Mon Aug 11 14:12:51 2014 +0200
@@ -2520,6 +2520,21 @@
set_control(norm);
}
+static IfNode* gen_subtype_check_compare(Node* ctrl, Node* in1, Node* in2, BoolTest::mask test, float p, PhaseGVN* gvn, BasicType bt) {
+ Node* cmp = NULL;
+ switch(bt) {
+ case T_INT: cmp = new CmpINode(in1, in2); break;
+ case T_ADDRESS: cmp = new CmpPNode(in1, in2); break;
+ default: fatal(err_msg("unexpected comparison type %s", type2name(bt)));
+ }
+ gvn->transform(cmp);
+ Node* bol = gvn->transform(new BoolNode(cmp, test));
+ IfNode* iff = new IfNode(ctrl, bol, p, COUNT_UNKNOWN);
+ gvn->transform(iff);
+ if (!bol->is_Con()) gvn->record_for_igvn(iff);
+ return iff;
+}
+
//-------------------------------gen_subtype_check-----------------------------
// Generate a subtyping check. Takes as input the subtype and supertype.
@@ -2529,16 +2544,17 @@
// but that's not exposed to the optimizer. This call also doesn't take in an
// Object; if you wish to check an Object you need to load the Object's class
// prior to coming here.
-Node* GraphKit::gen_subtype_check(Node* subklass, Node* superklass) {
+Node* Phase::gen_subtype_check(Node* subklass, Node* superklass, Node** ctrl, MergeMemNode* mem, PhaseGVN* gvn) {
+ Compile* C = gvn->C;
// Fast check for identical types, perhaps identical constants.
// The types can even be identical non-constants, in cases
// involving Array.newInstance, Object.clone, etc.
if (subklass == superklass)
- return top(); // false path is dead; no test needed.
-
- if (_gvn.type(superklass)->singleton()) {
- ciKlass* superk = _gvn.type(superklass)->is_klassptr()->klass();
- ciKlass* subk = _gvn.type(subklass)->is_klassptr()->klass();
+ return C->top(); // false path is dead; no test needed.
+
+ if (gvn->type(superklass)->singleton()) {
+ ciKlass* superk = gvn->type(superklass)->is_klassptr()->klass();
+ ciKlass* subk = gvn->type(subklass)->is_klassptr()->klass();
// In the common case of an exact superklass, try to fold up the
// test before generating code. You may ask, why not just generate
@@ -2549,25 +2565,23 @@
// Foo[] fa = blah(); Foo x = fa[0]; fa[1] = x;
// Here, the type of 'fa' is often exact, so the store check
// of fa[1]=x will fold up, without testing the nullness of x.
- switch (static_subtype_check(superk, subk)) {
- case SSC_always_false:
+ switch (C->static_subtype_check(superk, subk)) {
+ case Compile::SSC_always_false:
{
- Node* always_fail = control();
- set_control(top());
+ Node* always_fail = *ctrl;
+ *ctrl = gvn->C->top();
return always_fail;
}
- case SSC_always_true:
- return top();
- case SSC_easy_test:
+ case Compile::SSC_always_true:
+ return C->top();
+ case Compile::SSC_easy_test:
{
// Just do a direct pointer compare and be done.
- Node* cmp = _gvn.transform( new CmpPNode(subklass, superklass) );
- Node* bol = _gvn.transform( new BoolNode(cmp, BoolTest::eq) );
- IfNode* iff = create_and_xform_if(control(), bol, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
- set_control( _gvn.transform( new IfTrueNode (iff) ) );
- return _gvn.transform( new IfFalseNode(iff) );
+ IfNode* iff = gen_subtype_check_compare(*ctrl, subklass, superklass, BoolTest::eq, PROB_STATIC_FREQUENT, gvn, T_ADDRESS);
+ *ctrl = gvn->transform(new IfTrueNode(iff));
+ return gvn->transform(new IfFalseNode(iff));
}
- case SSC_full_test:
+ case Compile::SSC_full_test:
break;
default:
ShouldNotReachHere();
@@ -2579,11 +2593,11 @@
// will always succeed. We could leave a dependency behind to ensure this.
// First load the super-klass's check-offset
- Node *p1 = basic_plus_adr( superklass, superklass, in_bytes(Klass::super_check_offset_offset()) );
- Node *chk_off = _gvn.transform(new LoadINode(NULL, memory(p1), p1, _gvn.type(p1)->is_ptr(),
- TypeInt::INT, MemNode::unordered));
+ Node *p1 = gvn->transform(new AddPNode(superklass, superklass, gvn->MakeConX(in_bytes(Klass::super_check_offset_offset()))));
+ Node* m = mem->memory_at(C->get_alias_index(gvn->type(p1)->is_ptr()));
+ Node *chk_off = gvn->transform(new LoadINode(NULL, m, p1, gvn->type(p1)->is_ptr(), TypeInt::INT, MemNode::unordered));
int cacheoff_con = in_bytes(Klass::secondary_super_cache_offset());
- bool might_be_cache = (find_int_con(chk_off, cacheoff_con) == cacheoff_con);
+ bool might_be_cache = (gvn->find_int_con(chk_off, cacheoff_con) == cacheoff_con);
// Load from the sub-klass's super-class display list, or a 1-word cache of
// the secondary superclass list, or a failing value with a sentinel offset
@@ -2591,42 +2605,44 @@
// hierarchy and we have to scan the secondary superclass list the hard way.
// Worst-case type is a little odd: NULL is allowed as a result (usually
// klass loads can never produce a NULL).
- Node *chk_off_X = ConvI2X(chk_off);
- Node *p2 = _gvn.transform( new AddPNode(subklass,subklass,chk_off_X) );
+ Node *chk_off_X = chk_off;
+#ifdef _LP64
+ chk_off_X = gvn->transform(new ConvI2LNode(chk_off_X));
+#endif
+ Node *p2 = gvn->transform(new AddPNode(subklass,subklass,chk_off_X));
// For some types like interfaces the following loadKlass is from a 1-word
// cache which is mutable so can't use immutable memory. Other
// types load from the super-class display table which is immutable.
- Node *kmem = might_be_cache ? memory(p2) : immutable_memory();
- Node *nkls = _gvn.transform( LoadKlassNode::make( _gvn, kmem, p2, _gvn.type(p2)->is_ptr(), TypeKlassPtr::OBJECT_OR_NULL ) );
+ m = mem->memory_at(C->get_alias_index(gvn->type(p2)->is_ptr()));
+ Node *kmem = might_be_cache ? m : C->immutable_memory();
+ Node *nkls = gvn->transform(LoadKlassNode::make(*gvn, kmem, p2, gvn->type(p2)->is_ptr(), TypeKlassPtr::OBJECT_OR_NULL));
// Compile speed common case: ARE a subtype and we canNOT fail
if( superklass == nkls )
- return top(); // false path is dead; no test needed.
+ return C->top(); // false path is dead; no test needed.
// See if we get an immediate positive hit. Happens roughly 83% of the
// time. Test to see if the value loaded just previously from the subklass
// is exactly the superklass.
- Node *cmp1 = _gvn.transform( new CmpPNode( superklass, nkls ) );
- Node *bol1 = _gvn.transform( new BoolNode( cmp1, BoolTest::eq ) );
- IfNode *iff1 = create_and_xform_if( control(), bol1, PROB_LIKELY(0.83f), COUNT_UNKNOWN );
- Node *iftrue1 = _gvn.transform( new IfTrueNode ( iff1 ) );
- set_control( _gvn.transform( new IfFalseNode( iff1 ) ) );
+ IfNode *iff1 = gen_subtype_check_compare(*ctrl, superklass, nkls, BoolTest::eq, PROB_LIKELY(0.83f), gvn, T_ADDRESS);
+ Node *iftrue1 = gvn->transform( new IfTrueNode (iff1));
+ *ctrl = gvn->transform(new IfFalseNode(iff1));
// Compile speed common case: Check for being deterministic right now. If
// chk_off is a constant and not equal to cacheoff then we are NOT a
// subklass. In this case we need exactly the 1 test above and we can
// return those results immediately.
if (!might_be_cache) {
- Node* not_subtype_ctrl = control();
- set_control(iftrue1); // We need exactly the 1 test above
+ Node* not_subtype_ctrl = *ctrl;
+ *ctrl = iftrue1; // We need exactly the 1 test above
return not_subtype_ctrl;
}
// Gather the various success & failures here
RegionNode *r_ok_subtype = new RegionNode(4);
- record_for_igvn(r_ok_subtype);
+ gvn->record_for_igvn(r_ok_subtype);
RegionNode *r_not_subtype = new RegionNode(3);
- record_for_igvn(r_not_subtype);
+ gvn->record_for_igvn(r_not_subtype);
r_ok_subtype->init_req(1, iftrue1);
@@ -2635,21 +2651,17 @@
// check-offset points into the subklass display list or the 1-element
// cache. If it points to the display (and NOT the cache) and the display
// missed then it's not a subtype.
- Node *cacheoff = _gvn.intcon(cacheoff_con);
- Node *cmp2 = _gvn.transform( new CmpINode( chk_off, cacheoff ) );
- Node *bol2 = _gvn.transform( new BoolNode( cmp2, BoolTest::ne ) );
- IfNode *iff2 = create_and_xform_if( control(), bol2, PROB_LIKELY(0.63f), COUNT_UNKNOWN );
- r_not_subtype->init_req(1, _gvn.transform( new IfTrueNode (iff2) ) );
- set_control( _gvn.transform( new IfFalseNode(iff2) ) );
+ Node *cacheoff = gvn->intcon(cacheoff_con);
+ IfNode *iff2 = gen_subtype_check_compare(*ctrl, chk_off, cacheoff, BoolTest::ne, PROB_LIKELY(0.63f), gvn, T_INT);
+ r_not_subtype->init_req(1, gvn->transform(new IfTrueNode (iff2)));
+ *ctrl = gvn->transform(new IfFalseNode(iff2));
// Check for self. Very rare to get here, but it is taken 1/3 the time.
// No performance impact (too rare) but allows sharing of secondary arrays
// which has some footprint reduction.
- Node *cmp3 = _gvn.transform( new CmpPNode( subklass, superklass ) );
- Node *bol3 = _gvn.transform( new BoolNode( cmp3, BoolTest::eq ) );
- IfNode *iff3 = create_and_xform_if( control(), bol3, PROB_LIKELY(0.36f), COUNT_UNKNOWN );
- r_ok_subtype->init_req(2, _gvn.transform( new IfTrueNode ( iff3 ) ) );
- set_control( _gvn.transform( new IfFalseNode( iff3 ) ) );
+ IfNode *iff3 = gen_subtype_check_compare(*ctrl, subklass, superklass, BoolTest::eq, PROB_LIKELY(0.36f), gvn, T_ADDRESS);
+ r_ok_subtype->init_req(2, gvn->transform(new IfTrueNode(iff3)));
+ *ctrl = gvn->transform(new IfFalseNode(iff3));
// -- Roads not taken here: --
// We could also have chosen to perform the self-check at the beginning
@@ -2672,68 +2684,16 @@
// out of line, and it can only improve I-cache density.
// The decision to inline or out-of-line this final check is platform
// dependent, and is found in the AD file definition of PartialSubtypeCheck.
- Node* psc = _gvn.transform(
- new PartialSubtypeCheckNode(control(), subklass, superklass) );
-
- Node *cmp4 = _gvn.transform( new CmpPNode( psc, null() ) );
- Node *bol4 = _gvn.transform( new BoolNode( cmp4, BoolTest::ne ) );
- IfNode *iff4 = create_and_xform_if( control(), bol4, PROB_FAIR, COUNT_UNKNOWN );
- r_not_subtype->init_req(2, _gvn.transform( new IfTrueNode (iff4) ) );
- r_ok_subtype ->init_req(3, _gvn.transform( new IfFalseNode(iff4) ) );
+ Node* psc = gvn->transform(
+ new PartialSubtypeCheckNode(*ctrl, subklass, superklass));
+
+ IfNode *iff4 = gen_subtype_check_compare(*ctrl, psc, gvn->zerocon(T_OBJECT), BoolTest::ne, PROB_FAIR, gvn, T_ADDRESS);
+ r_not_subtype->init_req(2, gvn->transform(new IfTrueNode (iff4)));
+ r_ok_subtype ->init_req(3, gvn->transform(new IfFalseNode(iff4)));
// Return false path; set default control to true path.
- set_control( _gvn.transform(r_ok_subtype) );
- return _gvn.transform(r_not_subtype);
-}
-
-//----------------------------static_subtype_check-----------------------------
-// Shortcut important common cases when superklass is exact:
-// (0) superklass is java.lang.Object (can occur in reflective code)
-// (1) subklass is already limited to a subtype of superklass => always ok
-// (2) subklass does not overlap with superklass => always fail
-// (3) superklass has NO subtypes and we can check with a simple compare.
-int GraphKit::static_subtype_check(ciKlass* superk, ciKlass* subk) {
- if (StressReflectiveCode) {
- return SSC_full_test; // Let caller generate the general case.
- }
-
- if (superk == env()->Object_klass()) {
- return SSC_always_true; // (0) this test cannot fail
- }
-
- ciType* superelem = superk;
- if (superelem->is_array_klass())
- superelem = superelem->as_array_klass()->base_element_type();
-
- if (!subk->is_interface()) { // cannot trust static interface types yet
- if (subk->is_subtype_of(superk)) {
- return SSC_always_true; // (1) false path dead; no dynamic test needed
- }
- if (!(superelem->is_klass() && superelem->as_klass()->is_interface()) &&
- !superk->is_subtype_of(subk)) {
- return SSC_always_false;
- }
- }
-
- // If casting to an instance klass, it must have no subtypes
- if (superk->is_interface()) {
- // Cannot trust interfaces yet.
- // %%% S.B. superk->nof_implementors() == 1
- } else if (superelem->is_instance_klass()) {
- ciInstanceKlass* ik = superelem->as_instance_klass();
- if (!ik->has_subklass() && !ik->is_interface()) {
- if (!ik->is_final()) {
- // Add a dependency if there is a chance of a later subclass.
- C->dependencies()->assert_leaf_type(ik);
- }
- return SSC_easy_test; // (3) caller can do a simple ptr comparison
- }
- } else {
- // A primitive array type has no subtypes.
- return SSC_easy_test; // (3) caller can do a simple ptr comparison
- }
-
- return SSC_full_test;
+ *ctrl = gvn->transform(r_ok_subtype);
+ return gvn->transform(r_not_subtype);
}
// Profile-driven exact type check:
@@ -2813,7 +2773,7 @@
ciKlass* exact_kls = spec_klass == NULL ? profile_has_unique_klass() : spec_klass;
if (exact_kls != NULL) {// no cast failures here
if (require_klass == NULL ||
- static_subtype_check(require_klass, exact_kls) == SSC_always_true) {
+ C->static_subtype_check(require_klass, exact_kls) == Compile::SSC_always_true) {
// If we narrow the type to match what the type profile sees or
// the speculative type, we can then remove the rest of the
// cast.
@@ -2833,7 +2793,7 @@
}
return exact_obj;
}
- // assert(ssc == SSC_always_true)... except maybe the profile lied to us.
+ // assert(ssc == Compile::SSC_always_true)... except maybe the profile lied to us.
}
return NULL;
@@ -2938,8 +2898,8 @@
ciKlass* superk = _gvn.type(superklass)->is_klassptr()->klass();
ciKlass* subk = _gvn.type(obj)->is_oopptr()->klass();
if (subk != NULL && subk->is_loaded()) {
- int static_res = static_subtype_check(superk, subk);
- known_statically = (static_res == SSC_always_true || static_res == SSC_always_false);
+ int static_res = C->static_subtype_check(superk, subk);
+ known_statically = (static_res == Compile::SSC_always_true || static_res == Compile::SSC_always_false);
}
}
@@ -3007,13 +2967,13 @@
if (tk->singleton()) {
const TypeOopPtr* objtp = _gvn.type(obj)->isa_oopptr();
if (objtp != NULL && objtp->klass() != NULL) {
- switch (static_subtype_check(tk->klass(), objtp->klass())) {
- case SSC_always_true:
+ switch (C->static_subtype_check(tk->klass(), objtp->klass())) {
+ case Compile::SSC_always_true:
// If we know the type check always succeed then we don't use
// the profiling data at this bytecode. Don't lose it, feed it
// to the type system as a speculative type.
return record_profiled_receiver_for_speculation(obj);
- case SSC_always_false:
+ case Compile::SSC_always_false:
// It needs a null check because a null will *pass* the cast check.
// A non-null value will always produce an exception.
return null_assert(obj);
--- a/hotspot/src/share/vm/opto/graphKit.hpp Fri Aug 08 11:36:48 2014 -0700
+++ b/hotspot/src/share/vm/opto/graphKit.hpp Mon Aug 11 14:12:51 2014 +0200
@@ -829,17 +829,13 @@
Node* gen_checkcast( Node *subobj, Node* superkls,
Node* *failure_control = NULL );
- // Generate a subtyping check. Takes as input the subtype and supertype.
- // Returns 2 values: sets the default control() to the true path and
- // returns the false path. Only reads from constant memory taken from the
- // default memory; does not write anything. It also doesn't take in an
- // Object; if you wish to check an Object you need to load the Object's
- // class prior to coming here.
- Node* gen_subtype_check(Node* subklass, Node* superklass);
-
- // Static parse-time type checking logic for gen_subtype_check:
- enum { SSC_always_false, SSC_always_true, SSC_easy_test, SSC_full_test };
- int static_subtype_check(ciKlass* superk, ciKlass* subk);
+ Node* gen_subtype_check(Node* subklass, Node* superklass) {
+ MergeMemNode* mem = merged_memory();
+ Node* ctrl = control();
+ Node* n = Phase::gen_subtype_check(subklass, superklass, &ctrl, mem, &_gvn);
+ set_control(ctrl);
+ return n;
+ }
// Exact type check used for predicted calls and casts.
// Rewrites (*casted_receiver) to be casted to the stronger type.
--- a/hotspot/src/share/vm/opto/library_call.cpp Fri Aug 08 11:36:48 2014 -0700
+++ b/hotspot/src/share/vm/opto/library_call.cpp Mon Aug 11 14:12:51 2014 +0200
@@ -146,15 +146,10 @@
Node* generate_negative_guard(Node* index, RegionNode* region,
// resulting CastII of index:
Node* *pos_index = NULL);
- Node* generate_nonpositive_guard(Node* index, bool never_negative,
- // resulting CastII of index:
- Node* *pos_index = NULL);
Node* generate_limit_guard(Node* offset, Node* subseq_length,
Node* array_length,
RegionNode* region);
Node* generate_current_thread(Node* &tls_output);
- address basictype2arraycopy(BasicType t, Node *src_offset, Node *dest_offset,
- bool disjoint_bases, const char* &name, bool dest_uninitialized);
Node* load_mirror_from_klass(Node* klass);
Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null,
RegionNode* region, int null_path,
@@ -264,47 +259,8 @@
// Helper functions for inlining arraycopy
bool inline_arraycopy();
- void generate_arraycopy(const TypePtr* adr_type,
- BasicType basic_elem_type,
- Node* src, Node* src_offset,
- Node* dest, Node* dest_offset,
- Node* copy_length,
- bool disjoint_bases = false,
- bool length_never_negative = false,
- RegionNode* slow_region = NULL);
AllocateArrayNode* tightly_coupled_allocation(Node* ptr,
RegionNode* slow_region);
- void generate_clear_array(const TypePtr* adr_type,
- Node* dest,
- BasicType basic_elem_type,
- Node* slice_off,
- Node* slice_len,
- Node* slice_end);
- bool generate_block_arraycopy(const TypePtr* adr_type,
- BasicType basic_elem_type,
- AllocateNode* alloc,
- Node* src, Node* src_offset,
- Node* dest, Node* dest_offset,
- Node* dest_size, bool dest_uninitialized);
- void generate_slow_arraycopy(const TypePtr* adr_type,
- Node* src, Node* src_offset,
- Node* dest, Node* dest_offset,
- Node* copy_length, bool dest_uninitialized);
- Node* generate_checkcast_arraycopy(const TypePtr* adr_type,
- Node* dest_elem_klass,
- Node* src, Node* src_offset,
- Node* dest, Node* dest_offset,
- Node* copy_length, bool dest_uninitialized);
- Node* generate_generic_arraycopy(const TypePtr* adr_type,
- Node* src, Node* src_offset,
- Node* dest, Node* dest_offset,
- Node* copy_length, bool dest_uninitialized);
- void generate_unchecked_arraycopy(const TypePtr* adr_type,
- BasicType basic_elem_type,
- bool disjoint_bases,
- Node* src, Node* src_offset,
- Node* dest, Node* dest_offset,
- Node* copy_length, bool dest_uninitialized);
typedef enum { LS_xadd, LS_xchg, LS_cmpxchg } LoadStoreKind;
bool inline_unsafe_load_store(BasicType type, LoadStoreKind kind);
bool inline_unsafe_ordered_store(BasicType type);
@@ -1049,25 +1005,6 @@
return is_neg;
}
-inline Node* LibraryCallKit::generate_nonpositive_guard(Node* index, bool never_negative,
- Node* *pos_index) {
- if (stopped())
- return NULL; // already stopped
- if (_gvn.type(index)->higher_equal(TypeInt::POS1)) // [1,maxint]
- return NULL; // index is already adequately typed
- Node* cmp_le = _gvn.transform(new CmpINode(index, intcon(0)));
- BoolTest::mask le_or_eq = (never_negative ? BoolTest::eq : BoolTest::le);
- Node* bol_le = _gvn.transform(new BoolNode(cmp_le, le_or_eq));
- Node* is_notp = generate_guard(bol_le, NULL, PROB_MIN);
- if (is_notp != NULL && pos_index != NULL) {
- // Emulate effect of Parse::adjust_map_after_if.
- Node* ccast = new CastIINode(index, TypeInt::POS1);
- ccast->set_req(0, control());
- (*pos_index) = _gvn.transform(ccast);
- }
- return is_notp;
-}
-
// Make sure that 'position' is a valid limit index, in [0..length].
// There are two equivalent plans for checking this:
// A. (offset + copyLength) unsigned<= arrayLength
@@ -3928,13 +3865,18 @@
// oop stores need checking.
// Extreme case: Arrays.copyOf((Integer[])x, 10, String[].class).
// This will fail a store-check if x contains any non-nulls.
- bool disjoint_bases = true;
- // if start > orig_length then the length of the copy may be
- // negative.
- bool length_never_negative = !is_copyOfRange;
- generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT,
- original, start, newcopy, intcon(0), moved,
- disjoint_bases, length_never_negative);
+
+ Node* alloc = tightly_coupled_allocation(newcopy, NULL);
+
+ ArrayCopyNode* ac = ArrayCopyNode::make(this, true, original, start, newcopy, intcon(0), moved, alloc != NULL);
+ if (!is_copyOfRange) {
+ ac->set_copyof();
+ } else {
+ ac->set_copyofrange();
+ }
+ Node* n = _gvn.transform(ac);
+ assert(n == ac, "cannot disappear");
+ ac->connect_outputs(this);
}
} // original reexecute is set back here
@@ -4445,10 +4387,12 @@
countx = _gvn.transform(new URShiftXNode(countx, intcon(LogBytesPerLong) ));
const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
- bool disjoint_bases = true;
- generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases,
- src, NULL, dest, NULL, countx,
- /*dest_uninitialized*/true);
+
+ ArrayCopyNode* ac = ArrayCopyNode::make(this, false, src, NULL, dest, NULL, countx, false);
+ ac->set_clonebasic();
+ Node* n = _gvn.transform(ac);
+ assert(n == ac, "cannot disappear");
+ set_predefined_output_for_runtime_call(ac, ac->in(TypeFunc::Memory), raw_adr_type);
// If necessary, emit some card marks afterwards. (Non-arrays only.)
if (card_mark) {
@@ -4557,12 +4501,13 @@
PreserveJVMState pjvms2(this);
set_control(is_obja);
// Generate a direct call to the right arraycopy function(s).
- bool disjoint_bases = true;
- bool length_never_negative = true;
- generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT,
- obj, intcon(0), alloc_obj, intcon(0),
- obj_length,
- disjoint_bases, length_never_negative);
+ Node* alloc = tightly_coupled_allocation(alloc_obj, NULL);
+ ArrayCopyNode* ac = ArrayCopyNode::make(this, true, obj, intcon(0), alloc_obj, intcon(0), obj_length, alloc != NULL);
+ ac->set_cloneoop();
+ Node* n = _gvn.transform(ac);
+ assert(n == ac, "cannot disappear");
+ ac->connect_outputs(this);
+
result_reg->init_req(_objArray_path, control());
result_val->init_req(_objArray_path, alloc_obj);
result_i_o ->set_req(_objArray_path, i_o());
@@ -4656,42 +4601,6 @@
return true;
}
-//------------------------------basictype2arraycopy----------------------------
-address LibraryCallKit::basictype2arraycopy(BasicType t,
- Node* src_offset,
- Node* dest_offset,
- bool disjoint_bases,
- const char* &name,
- bool dest_uninitialized) {
- const TypeInt* src_offset_inttype = gvn().find_int_type(src_offset);;
- const TypeInt* dest_offset_inttype = gvn().find_int_type(dest_offset);;
-
- bool aligned = false;
- bool disjoint = disjoint_bases;
-
- // if the offsets are the same, we can treat the memory regions as
- // disjoint, because either the memory regions are in different arrays,
- // or they are identical (which we can treat as disjoint.) We can also
- // treat a copy with a destination index less that the source index
- // as disjoint since a low->high copy will work correctly in this case.
- if (src_offset_inttype != NULL && src_offset_inttype->is_con() &&
- dest_offset_inttype != NULL && dest_offset_inttype->is_con()) {
- // both indices are constants
- int s_offs = src_offset_inttype->get_con();
- int d_offs = dest_offset_inttype->get_con();
- int element_size = type2aelembytes(t);
- aligned = ((arrayOopDesc::base_offset_in_bytes(t) + s_offs * element_size) % HeapWordSize == 0) &&
- ((arrayOopDesc::base_offset_in_bytes(t) + d_offs * element_size) % HeapWordSize == 0);
- if (s_offs >= d_offs) disjoint = true;
- } else if (src_offset == dest_offset && src_offset != NULL) {
- // This can occur if the offsets are identical non-constants.
- disjoint = true;
- }
-
- return StubRoutines::select_arraycopy_function(t, aligned, disjoint, name, dest_uninitialized);
-}
-
-
//------------------------------inline_arraycopy-----------------------
// public static native void java.lang.System.arraycopy(Object src, int srcPos,
// Object dest, int destPos,
@@ -4704,13 +4613,26 @@
Node* dest_offset = argument(3); // type: int
Node* length = argument(4); // type: int
- // Compile time checks. If any of these checks cannot be verified at compile time,
- // we do not make a fast path for this call. Instead, we let the call remain as it
- // is. The checks we choose to mandate at compile time are:
- //
+ // The following tests must be performed
// (1) src and dest are arrays.
- const Type* src_type = src->Value(&_gvn);
- const Type* dest_type = dest->Value(&_gvn);
+ // (2) src and dest arrays must have elements of the same BasicType
+ // (3) src and dest must not be null.
+ // (4) src_offset must not be negative.
+ // (5) dest_offset must not be negative.
+ // (6) length must not be negative.
+ // (7) src_offset + length must not exceed length of src.
+ // (8) dest_offset + length must not exceed length of dest.
+ // (9) each element of an oop array must be assignable
+
+ // (3) src and dest must not be null.
+ // always do this here because we need the JVM state for uncommon traps
+ src = null_check(src, T_ARRAY);
+ dest = null_check(dest, T_ARRAY);
+
+ bool notest = false;
+
+ const Type* src_type = _gvn.type(src);
+ const Type* dest_type = _gvn.type(dest);
const TypeAryPtr* top_src = src_type->isa_aryptr();
const TypeAryPtr* top_dest = dest_type->isa_aryptr();
@@ -4768,556 +4690,114 @@
}
}
- if (!has_src || !has_dest) {
- // Conservatively insert a memory barrier on all memory slices.
- // Do not let writes into the source float below the arraycopy.
- insert_mem_bar(Op_MemBarCPUOrder);
-
- // Call StubRoutines::generic_arraycopy stub.
- generate_arraycopy(TypeRawPtr::BOTTOM, T_CONFLICT,
- src, src_offset, dest, dest_offset, length);
-
- // Do not let reads from the destination float above the arraycopy.
- // Since we cannot type the arrays, we don't know which slices
- // might be affected. We could restrict this barrier only to those
- // memory slices which pertain to array elements--but don't bother.
- if (!InsertMemBarAfterArraycopy)
- // (If InsertMemBarAfterArraycopy, there is already one in place.)
- insert_mem_bar(Op_MemBarCPUOrder);
- return true;
- }
-
- // (2) src and dest arrays must have elements of the same BasicType
- // Figure out the size and type of the elements we will be copying.
- BasicType src_elem = top_src->klass()->as_array_klass()->element_type()->basic_type();
- BasicType dest_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type();
- if (src_elem == T_ARRAY) src_elem = T_OBJECT;
- if (dest_elem == T_ARRAY) dest_elem = T_OBJECT;
-
- if (src_elem != dest_elem || dest_elem == T_VOID) {
- // The component types are not the same or are not recognized. Punt.
- // (But, avoid the native method wrapper to JVM_ArrayCopy.)
- generate_slow_arraycopy(TypePtr::BOTTOM,
- src, src_offset, dest, dest_offset, length,
- /*dest_uninitialized*/false);
- return true;
- }
-
- if (src_elem == T_OBJECT) {
- // If both arrays are object arrays then having the exact types
- // for both will remove the need for a subtype check at runtime
- // before the call and may make it possible to pick a faster copy
- // routine (without a subtype check on every element)
- // Do we have the exact type of src?
- bool could_have_src = src_spec;
- // Do we have the exact type of dest?
- bool could_have_dest = dest_spec;
- ciKlass* src_k = top_src->klass();
- ciKlass* dest_k = top_dest->klass();
- if (!src_spec) {
- src_k = src_type->speculative_type_not_null();
- if (src_k != NULL && src_k->is_array_klass()) {
+ if (has_src && has_dest) {
+ BasicType src_elem = top_src->klass()->as_array_klass()->element_type()->basic_type();
+ BasicType dest_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type();
+ if (src_elem == T_ARRAY) src_elem = T_OBJECT;
+ if (dest_elem == T_ARRAY) dest_elem = T_OBJECT;
+
+ if (src_elem == dest_elem && src_elem == T_OBJECT) {
+ // If both arrays are object arrays then having the exact types
+ // for both will remove the need for a subtype check at runtime
+ // before the call and may make it possible to pick a faster copy
+ // routine (without a subtype check on every element)
+ // Do we have the exact type of src?
+ bool could_have_src = src_spec;
+ // Do we have the exact type of dest?
+ bool could_have_dest = dest_spec;
+ ciKlass* src_k = top_src->klass();
+ ciKlass* dest_k = top_dest->klass();
+ if (!src_spec) {
+ src_k = src_type->speculative_type_not_null();
+ if (src_k != NULL && src_k->is_array_klass()) {
could_have_src = true;
+ }
}
- }
- if (!dest_spec) {
- dest_k = dest_type->speculative_type_not_null();
- if (dest_k != NULL && dest_k->is_array_klass()) {
- could_have_dest = true;
+ if (!dest_spec) {
+ dest_k = dest_type->speculative_type_not_null();
+ if (dest_k != NULL && dest_k->is_array_klass()) {
+ could_have_dest = true;
+ }
}
- }
- if (could_have_src && could_have_dest) {
- // If we can have both exact types, emit the missing guards
- if (could_have_src && !src_spec) {
- src = maybe_cast_profiled_obj(src, src_k);
- }
- if (could_have_dest && !dest_spec) {
- dest = maybe_cast_profiled_obj(dest, dest_k);
+ if (could_have_src && could_have_dest) {
+ // If we can have both exact types, emit the missing guards
+ if (could_have_src && !src_spec) {
+ src = maybe_cast_profiled_obj(src, src_k);
+ }
+ if (could_have_dest && !dest_spec) {
+ dest = maybe_cast_profiled_obj(dest, dest_k);
+ }
}
}
}
- //---------------------------------------------------------------------------
- // We will make a fast path for this call to arraycopy.
-
- // We have the following tests left to perform:
- //
- // (3) src and dest must not be null.
- // (4) src_offset must not be negative.
- // (5) dest_offset must not be negative.
- // (6) length must not be negative.
- // (7) src_offset + length must not exceed length of src.
- // (8) dest_offset + length must not exceed length of dest.
- // (9) each element of an oop array must be assignable
-
- RegionNode* slow_region = new RegionNode(1);
- record_for_igvn(slow_region);
-
- // (3) operands must not be null
- // We currently perform our null checks with the null_check routine.
- // This means that the null exceptions will be reported in the caller
- // rather than (correctly) reported inside of the native arraycopy call.
- // This should be corrected, given time. We do our null check with the
- // stack pointer restored.
- src = null_check(src, T_ARRAY);
- dest = null_check(dest, T_ARRAY);
-
- // (4) src_offset must not be negative.
- generate_negative_guard(src_offset, slow_region);
-
- // (5) dest_offset must not be negative.
- generate_negative_guard(dest_offset, slow_region);
-
- // (6) length must not be negative (moved to generate_arraycopy()).
- // generate_negative_guard(length, slow_region);
-
- // (7) src_offset + length must not exceed length of src.
- generate_limit_guard(src_offset, length,
- load_array_length(src),
- slow_region);
-
- // (8) dest_offset + length must not exceed length of dest.
- generate_limit_guard(dest_offset, length,
- load_array_length(dest),
- slow_region);
-
- // (9) each element of an oop array must be assignable
- // The generate_arraycopy subroutine checks this.
-
- // This is where the memory effects are placed:
- const TypePtr* adr_type = TypeAryPtr::get_array_body_type(dest_elem);
- generate_arraycopy(adr_type, dest_elem,
- src, src_offset, dest, dest_offset, length,
- false, false, slow_region);
-
- return true;
-}
-
-//-----------------------------generate_arraycopy----------------------
-// Generate an optimized call to arraycopy.
-// Caller must guard against non-arrays.
-// Caller must determine a common array basic-type for both arrays.
-// Caller must validate offsets against array bounds.
-// The slow_region has already collected guard failure paths
-// (such as out of bounds length or non-conformable array types).
-// The generated code has this shape, in general:
-//
-// if (length == 0) return // via zero_path
-// slowval = -1
-// if (types unknown) {
-// slowval = call generic copy loop
-// if (slowval == 0) return // via checked_path
-// } else if (indexes in bounds) {
-// if ((is object array) && !(array type check)) {
-// slowval = call checked copy loop
-// if (slowval == 0) return // via checked_path
-// } else {
-// call bulk copy loop
-// return // via fast_path
-// }
-// }
-// // adjust params for remaining work:
-// if (slowval != -1) {
-// n = -1^slowval; src_offset += n; dest_offset += n; length -= n
-// }
-// slow_region:
-// call slow arraycopy(src, src_offset, dest, dest_offset, length)
-// return // via slow_call_path
-//
-// This routine is used from several intrinsics: System.arraycopy,
-// Object.clone (the array subcase), and Arrays.copyOf[Range].
-//
-void
-LibraryCallKit::generate_arraycopy(const TypePtr* adr_type,
- BasicType basic_elem_type,
- Node* src, Node* src_offset,
- Node* dest, Node* dest_offset,
- Node* copy_length,
- bool disjoint_bases,
- bool length_never_negative,
- RegionNode* slow_region) {
-
- if (slow_region == NULL) {
- slow_region = new RegionNode(1);
+ if (!too_many_traps(Deoptimization::Reason_intrinsic) && !src->is_top() && !dest->is_top()) {
+ // validate arguments: enables transformation the ArrayCopyNode
+ notest = true;
+
+ RegionNode* slow_region = new RegionNode(1);
record_for_igvn(slow_region);
- }
-
- Node* original_dest = dest;
- AllocateArrayNode* alloc = NULL; // used for zeroing, if needed
- bool dest_uninitialized = false;
-
- // See if this is the initialization of a newly-allocated array.
- // If so, we will take responsibility here for initializing it to zero.
- // (Note: Because tightly_coupled_allocation performs checks on the
- // out-edges of the dest, we need to avoid making derived pointers
- // from it until we have checked its uses.)
- if (ReduceBulkZeroing
- && !ZeroTLAB // pointless if already zeroed
- && basic_elem_type != T_CONFLICT // avoid corner case
- && !src->eqv_uncast(dest)
- && ((alloc = tightly_coupled_allocation(dest, slow_region))
- != NULL)
- && _gvn.find_int_con(alloc->in(AllocateNode::ALength), 1) > 0
- && alloc->maybe_set_complete(&_gvn)) {
- // "You break it, you buy it."
- InitializeNode* init = alloc->initialization();
- assert(init->is_complete(), "we just did this");
- init->set_complete_with_arraycopy();
- assert(dest->is_CheckCastPP(), "sanity");
- assert(dest->in(0)->in(0) == init, "dest pinned");
- adr_type = TypeRawPtr::BOTTOM; // all initializations are into raw memory
- // From this point on, every exit path is responsible for
- // initializing any non-copied parts of the object to zero.
- // Also, if this flag is set we make sure that arraycopy interacts properly
- // with G1, eliding pre-barriers. See CR 6627983.
- dest_uninitialized = true;
- } else {
- // No zeroing elimination here.
- alloc = NULL;
- //original_dest = dest;
- //dest_uninitialized = false;
- }
-
- // Results are placed here:
- enum { fast_path = 1, // normal void-returning assembly stub
- checked_path = 2, // special assembly stub with cleanup
- slow_call_path = 3, // something went wrong; call the VM
- zero_path = 4, // bypass when length of copy is zero
- bcopy_path = 5, // copy primitive array by 64-bit blocks
- PATH_LIMIT = 6
- };
- RegionNode* result_region = new RegionNode(PATH_LIMIT);
- PhiNode* result_i_o = new PhiNode(result_region, Type::ABIO);
- PhiNode* result_memory = new PhiNode(result_region, Type::MEMORY, adr_type);
- record_for_igvn(result_region);
- _gvn.set_type_bottom(result_i_o);
- _gvn.set_type_bottom(result_memory);
- assert(adr_type != TypePtr::BOTTOM, "must be RawMem or a T[] slice");
-
- // The slow_control path:
- Node* slow_control;
- Node* slow_i_o = i_o();
- Node* slow_mem = memory(adr_type);
- debug_only(slow_control = (Node*) badAddress);
-
- // Checked control path:
- Node* checked_control = top();
- Node* checked_mem = NULL;
- Node* checked_i_o = NULL;
- Node* checked_value = NULL;
-
- if (basic_elem_type == T_CONFLICT) {
- assert(!dest_uninitialized, "");
- Node* cv = generate_generic_arraycopy(adr_type,
- src, src_offset, dest, dest_offset,
- copy_length, dest_uninitialized);
- if (cv == NULL) cv = intcon(-1); // failure (no stub available)
- checked_control = control();
- checked_i_o = i_o();
- checked_mem = memory(adr_type);
- checked_value = cv;
- set_control(top()); // no fast path
- }
-
- Node* not_pos = generate_nonpositive_guard(copy_length, length_never_negative);
- if (not_pos != NULL) {
- PreserveJVMState pjvms(this);
- set_control(not_pos);
-
- // (6) length must not be negative.
- if (!length_never_negative) {
- generate_negative_guard(copy_length, slow_region);
- }
-
- // copy_length is 0.
- if (!stopped() && dest_uninitialized) {
- Node* dest_length = alloc->in(AllocateNode::ALength);
- if (copy_length->eqv_uncast(dest_length)
- || _gvn.find_int_con(dest_length, 1) <= 0) {
- // There is no zeroing to do. No need for a secondary raw memory barrier.
- } else {
- // Clear the whole thing since there are no source elements to copy.
- generate_clear_array(adr_type, dest, basic_elem_type,
- intcon(0), NULL,
- alloc->in(AllocateNode::AllocSize));
- // Use a secondary InitializeNode as raw memory barrier.
- // Currently it is needed only on this path since other
- // paths have stub or runtime calls as raw memory barriers.
- InitializeNode* init = insert_mem_bar_volatile(Op_Initialize,
- Compile::AliasIdxRaw,
- top())->as_Initialize();
- init->set_complete(&_gvn); // (there is no corresponding AllocateNode)
- }
- }
-
- // Present the results of the fast call.
- result_region->init_req(zero_path, control());
- result_i_o ->init_req(zero_path, i_o());
- result_memory->init_req(zero_path, memory(adr_type));
- }
-
- if (!stopped() && dest_uninitialized) {
- // We have to initialize the *uncopied* part of the array to zero.
- // The copy destination is the slice dest[off..off+len]. The other slices
- // are dest_head = dest[0..off] and dest_tail = dest[off+len..dest.length].
- Node* dest_size = alloc->in(AllocateNode::AllocSize);
- Node* dest_length = alloc->in(AllocateNode::ALength);
- Node* dest_tail = _gvn.transform(new AddINode(dest_offset, copy_length));
-
- // If there is a head section that needs zeroing, do it now.
- if (find_int_con(dest_offset, -1) != 0) {
- generate_clear_array(adr_type, dest, basic_elem_type,
- intcon(0), dest_offset,
- NULL);
- }
-
- // Next, perform a dynamic check on the tail length.
- // It is often zero, and we can win big if we prove this.
- // There are two wins: Avoid generating the ClearArray
- // with its attendant messy index arithmetic, and upgrade
- // the copy to a more hardware-friendly word size of 64 bits.
- Node* tail_ctl = NULL;
- if (!stopped() && !dest_tail->eqv_uncast(dest_length)) {
- Node* cmp_lt = _gvn.transform(new CmpINode(dest_tail, dest_length));
- Node* bol_lt = _gvn.transform(new BoolNode(cmp_lt, BoolTest::lt));
- tail_ctl = generate_slow_guard(bol_lt, NULL);
- assert(tail_ctl != NULL || !stopped(), "must be an outcome");
- }
-
- // At this point, let's assume there is no tail.
- if (!stopped() && alloc != NULL && basic_elem_type != T_OBJECT) {
- // There is no tail. Try an upgrade to a 64-bit copy.
- bool didit = false;
- { PreserveJVMState pjvms(this);
- didit = generate_block_arraycopy(adr_type, basic_elem_type, alloc,
- src, src_offset, dest, dest_offset,
- dest_size, dest_uninitialized);
- if (didit) {
- // Present the results of the block-copying fast call.
- result_region->init_req(bcopy_path, control());
- result_i_o ->init_req(bcopy_path, i_o());
- result_memory->init_req(bcopy_path, memory(adr_type));
- }
- }
- if (didit)
- set_control(top()); // no regular fast path
- }
-
- // Clear the tail, if any.
- if (tail_ctl != NULL) {
- Node* notail_ctl = stopped() ? NULL : control();
- set_control(tail_ctl);
- if (notail_ctl == NULL) {
- generate_clear_array(adr_type, dest, basic_elem_type,
- dest_tail, NULL,
- dest_size);
- } else {
- // Make a local merge.
- Node* done_ctl = new RegionNode(3);
- Node* done_mem = new PhiNode(done_ctl, Type::MEMORY, adr_type);
- done_ctl->init_req(1, notail_ctl);
- done_mem->init_req(1, memory(adr_type));
- generate_clear_array(adr_type, dest, basic_elem_type,
- dest_tail, NULL,
- dest_size);
- done_ctl->init_req(2, control());
- done_mem->init_req(2, memory(adr_type));
- set_control( _gvn.transform(done_ctl));
- set_memory( _gvn.transform(done_mem), adr_type );
- }
- }
- }
-
- BasicType copy_type = basic_elem_type;
- assert(basic_elem_type != T_ARRAY, "caller must fix this");
- if (!stopped() && copy_type == T_OBJECT) {
- // If src and dest have compatible element types, we can copy bits.
- // Types S[] and D[] are compatible if D is a supertype of S.
- //
- // If they are not, we will use checked_oop_disjoint_arraycopy,
- // which performs a fast optimistic per-oop check, and backs off
- // further to JVM_ArrayCopy on the first per-oop check that fails.
- // (Actually, we don't move raw bits only; the GC requires card marks.)
-
- // Get the Klass* for both src and dest
+
+ // (1) src and dest are arrays.
+ generate_non_array_guard(load_object_klass(src), slow_region);
+ generate_non_array_guard(load_object_klass(dest), slow_region);
+
+ // (2) src and dest arrays must have elements of the same BasicType
+ // done at macro expansion or at Ideal transformation time
+
+ // (4) src_offset must not be negative.
+ generate_negative_guard(src_offset, slow_region);
+
+ // (5) dest_offset must not be negative.
+ generate_negative_guard(dest_offset, slow_region);
+
+ // (7) src_offset + length must not exceed length of src.
+ generate_limit_guard(src_offset, length,
+ load_array_length(src),
+ slow_region);
+
+ // (8) dest_offset + length must not exceed length of dest.
+ generate_limit_guard(dest_offset, length,
+ load_array_length(dest),
+ slow_region);
+
+ // (9) each element of an oop array must be assignable
Node* src_klass = load_object_klass(src);
Node* dest_klass = load_object_klass(dest);
-
- // Generate the subtype check.
- // This might fold up statically, or then again it might not.
- //
- // Non-static example: Copying List<String>.elements to a new String[].
- // The backing store for a List<String> is always an Object[],
- // but its elements are always type String, if the generic types
- // are correct at the source level.
- //
- // Test S[] against D[], not S against D, because (probably)
- // the secondary supertype cache is less busy for S[] than S.
- // This usually only matters when D is an interface.
Node* not_subtype_ctrl = gen_subtype_check(src_klass, dest_klass);
- // Plug failing path into checked_oop_disjoint_arraycopy
+
if (not_subtype_ctrl != top()) {
PreserveJVMState pjvms(this);
set_control(not_subtype_ctrl);
- // (At this point we can assume disjoint_bases, since types differ.)
- int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
- Node* p1 = basic_plus_adr(dest_klass, ek_offset);
- Node* n1 = LoadKlassNode::make(_gvn, immutable_memory(), p1, TypeRawPtr::BOTTOM);
- Node* dest_elem_klass = _gvn.transform(n1);
- Node* cv = generate_checkcast_arraycopy(adr_type,
- dest_elem_klass,
- src, src_offset, dest, dest_offset,
- ConvI2X(copy_length), dest_uninitialized);
- if (cv == NULL) cv = intcon(-1); // failure (no stub available)
- checked_control = control();
- checked_i_o = i_o();
- checked_mem = memory(adr_type);
- checked_value = cv;
+ uncommon_trap(Deoptimization::Reason_intrinsic,
+ Deoptimization::Action_make_not_entrant);
+ assert(stopped(), "Should be stopped");
}
- // At this point we know we do not need type checks on oop stores.
-
- // Let's see if we need card marks:
- if (alloc != NULL && use_ReduceInitialCardMarks()) {
- // If we do not need card marks, copy using the jint or jlong stub.
- copy_type = LP64_ONLY(UseCompressedOops ? T_INT : T_LONG) NOT_LP64(T_INT);
- assert(type2aelembytes(basic_elem_type) == type2aelembytes(copy_type),
- "sizes agree");
+ {
+ PreserveJVMState pjvms(this);
+ set_control(_gvn.transform(slow_region));
+ uncommon_trap(Deoptimization::Reason_intrinsic,
+ Deoptimization::Action_make_not_entrant);
+ assert(stopped(), "Should be stopped");
}
}
- if (!stopped()) {
- // Generate the fast path, if possible.
- PreserveJVMState pjvms(this);
- generate_unchecked_arraycopy(adr_type, copy_type, disjoint_bases,
- src, src_offset, dest, dest_offset,
- ConvI2X(copy_length), dest_uninitialized);
-
- // Present the results of the fast call.
- result_region->init_req(fast_path, control());
- result_i_o ->init_req(fast_path, i_o());
- result_memory->init_req(fast_path, memory(adr_type));
+ if (stopped()) {
+ return true;
}
- // Here are all the slow paths up to this point, in one bundle:
- slow_control = top();
- if (slow_region != NULL)
- slow_control = _gvn.transform(slow_region);
- DEBUG_ONLY(slow_region = (RegionNode*)badAddress);
-
- set_control(checked_control);
- if (!stopped()) {
- // Clean up after the checked call.
- // The returned value is either 0 or -1^K,
- // where K = number of partially transferred array elements.
- Node* cmp = _gvn.transform(new CmpINode(checked_value, intcon(0)));
- Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::eq));
- IfNode* iff = create_and_map_if(control(), bol, PROB_MAX, COUNT_UNKNOWN);
-
- // If it is 0, we are done, so transfer to the end.
- Node* checks_done = _gvn.transform(new IfTrueNode(iff));
- result_region->init_req(checked_path, checks_done);
- result_i_o ->init_req(checked_path, checked_i_o);
- result_memory->init_req(checked_path, checked_mem);
-
- // If it is not zero, merge into the slow call.
- set_control( _gvn.transform(new IfFalseNode(iff) ));
- RegionNode* slow_reg2 = new RegionNode(3);
- PhiNode* slow_i_o2 = new PhiNode(slow_reg2, Type::ABIO);
- PhiNode* slow_mem2 = new PhiNode(slow_reg2, Type::MEMORY, adr_type);
- record_for_igvn(slow_reg2);
- slow_reg2 ->init_req(1, slow_control);
- slow_i_o2 ->init_req(1, slow_i_o);
- slow_mem2 ->init_req(1, slow_mem);
- slow_reg2 ->init_req(2, control());
- slow_i_o2 ->init_req(2, checked_i_o);
- slow_mem2 ->init_req(2, checked_mem);
-
- slow_control = _gvn.transform(slow_reg2);
- slow_i_o = _gvn.transform(slow_i_o2);
- slow_mem = _gvn.transform(slow_mem2);
-
- if (alloc != NULL) {
- // We'll restart from the very beginning, after zeroing the whole thing.
- // This can cause double writes, but that's OK since dest is brand new.
- // So we ignore the low 31 bits of the value returned from the stub.
- } else {
- // We must continue the copy exactly where it failed, or else
- // another thread might see the wrong number of writes to dest.
- Node* checked_offset = _gvn.transform(new XorINode(checked_value, intcon(-1)));
- Node* slow_offset = new PhiNode(slow_reg2, TypeInt::INT);
- slow_offset->init_req(1, intcon(0));
- slow_offset->init_req(2, checked_offset);
- slow_offset = _gvn.transform(slow_offset);
-
- // Adjust the arguments by the conditionally incoming offset.
- Node* src_off_plus = _gvn.transform(new AddINode(src_offset, slow_offset));
- Node* dest_off_plus = _gvn.transform(new AddINode(dest_offset, slow_offset));
- Node* length_minus = _gvn.transform(new SubINode(copy_length, slow_offset));
-
- // Tweak the node variables to adjust the code produced below:
- src_offset = src_off_plus;
- dest_offset = dest_off_plus;
- copy_length = length_minus;
- }
+ AllocateArrayNode* alloc = tightly_coupled_allocation(dest, NULL);
+ ArrayCopyNode* ac = ArrayCopyNode::make(this, true, src, src_offset, dest, dest_offset, length, alloc != NULL);
+
+ if (notest) {
+ ac->set_arraycopy_notest();
}
- set_control(slow_control);
- if (!stopped()) {
- // Generate the slow path, if needed.
- PreserveJVMState pjvms(this); // replace_in_map may trash the map
-
- set_memory(slow_mem, adr_type);
- set_i_o(slow_i_o);
-
- if (dest_uninitialized) {
- generate_clear_array(adr_type, dest, basic_elem_type,
- intcon(0), NULL,
- alloc->in(AllocateNode::AllocSize));
- }
-
- generate_slow_arraycopy(adr_type,
- src, src_offset, dest, dest_offset,
- copy_length, /*dest_uninitialized*/false);
-
- result_region->init_req(slow_call_path, control());
- result_i_o ->init_req(slow_call_path, i_o());
- result_memory->init_req(slow_call_path, memory(adr_type));
- }
-
- // Remove unused edges.
- for (uint i = 1; i < result_region->req(); i++) {
- if (result_region->in(i) == NULL)
- result_region->init_req(i, top());
- }
-
- // Finished; return the combined state.
- set_control( _gvn.transform(result_region));
- set_i_o( _gvn.transform(result_i_o) );
- set_memory( _gvn.transform(result_memory), adr_type );
-
- // The memory edges above are precise in order to model effects around
- // array copies accurately to allow value numbering of field loads around
- // arraycopy. Such field loads, both before and after, are common in Java
- // collections and similar classes involving header/array data structures.
- //
- // But with low number of register or when some registers are used or killed
- // by arraycopy calls it causes registers spilling on stack. See 6544710.
- // The next memory barrier is added to avoid it. If the arraycopy can be
- // optimized away (which it can, sometimes) then we can manually remove
- // the membar also.
- //
- // Do not let reads from the cloned object float above the arraycopy.
- if (alloc != NULL) {
- // Do not let stores that initialize this object be reordered with
- // a subsequent store that would make this object accessible by
- // other threads.
- // Record what AllocateNode this StoreStore protects so that
- // escape analysis can go from the MemBarStoreStoreNode to the
- // AllocateNode and eliminate the MemBarStoreStoreNode if possible
- // based on the escape status of the AllocateNode.
- insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
- } else if (InsertMemBarAfterArraycopy)
- insert_mem_bar(Op_MemBarCPUOrder);
+ Node* n = _gvn.transform(ac);
+ assert(n == ac, "cannot disappear");
+ ac->connect_outputs(this);
+
+ return true;
}
@@ -5397,305 +4877,6 @@
return alloc;
}
-// Helper for initialization of arrays, creating a ClearArray.
-// It writes zero bits in [start..end), within the body of an array object.
-// The memory effects are all chained onto the 'adr_type' alias category.
-//
-// Since the object is otherwise uninitialized, we are free
-// to put a little "slop" around the edges of the cleared area,
-// as long as it does not go back into the array's header,
-// or beyond the array end within the heap.
-//
-// The lower edge can be rounded down to the nearest jint and the
-// upper edge can be rounded up to the nearest MinObjAlignmentInBytes.
-//
-// Arguments:
-// adr_type memory slice where writes are generated
-// dest oop of the destination array
-// basic_elem_type element type of the destination
-// slice_idx array index of first element to store
-// slice_len number of elements to store (or NULL)
-// dest_size total size in bytes of the array object
-//
-// Exactly one of slice_len or dest_size must be non-NULL.
-// If dest_size is non-NULL, zeroing extends to the end of the object.
-// If slice_len is non-NULL, the slice_idx value must be a constant.
-void
-LibraryCallKit::generate_clear_array(const TypePtr* adr_type,
- Node* dest,
- BasicType basic_elem_type,
- Node* slice_idx,
- Node* slice_len,
- Node* dest_size) {
- // one or the other but not both of slice_len and dest_size:
- assert((slice_len != NULL? 1: 0) + (dest_size != NULL? 1: 0) == 1, "");
- if (slice_len == NULL) slice_len = top();
- if (dest_size == NULL) dest_size = top();
-
- // operate on this memory slice:
- Node* mem = memory(adr_type); // memory slice to operate on
-
- // scaling and rounding of indexes:
- int scale = exact_log2(type2aelembytes(basic_elem_type));
- int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
- int clear_low = (-1 << scale) & (BytesPerInt - 1);
- int bump_bit = (-1 << scale) & BytesPerInt;
-
- // determine constant starts and ends
- const intptr_t BIG_NEG = -128;
- assert(BIG_NEG + 2*abase < 0, "neg enough");
- intptr_t slice_idx_con = (intptr_t) find_int_con(slice_idx, BIG_NEG);
- intptr_t slice_len_con = (intptr_t) find_int_con(slice_len, BIG_NEG);
- if (slice_len_con == 0) {
- return; // nothing to do here
- }
- intptr_t start_con = (abase + (slice_idx_con << scale)) & ~clear_low;
- intptr_t end_con = find_intptr_t_con(dest_size, -1);
- if (slice_idx_con >= 0 && slice_len_con >= 0) {
- assert(end_con < 0, "not two cons");
- end_con = round_to(abase + ((slice_idx_con + slice_len_con) << scale),
- BytesPerLong);
- }
-
- if (start_con >= 0 && end_con >= 0) {
- // Constant start and end. Simple.
- mem = ClearArrayNode::clear_memory(control(), mem, dest,
- start_con, end_con, &_gvn);
- } else if (start_con >= 0 && dest_size != top()) {
- // Constant start, pre-rounded end after the tail of the array.
- Node* end = dest_size;
- mem = ClearArrayNode::clear_memory(control(), mem, dest,
- start_con, end, &_gvn);
- } else if (start_con >= 0 && slice_len != top()) {
- // Constant start, non-constant end. End needs rounding up.
- // End offset = round_up(abase + ((slice_idx_con + slice_len) << scale), 8)
- intptr_t end_base = abase + (slice_idx_con << scale);
- int end_round = (-1 << scale) & (BytesPerLong - 1);
- Node* end = ConvI2X(slice_len);
- if (scale != 0)
- end = _gvn.transform(new LShiftXNode(end, intcon(scale) ));
- end_base += end_round;
- end = _gvn.transform(new AddXNode(end, MakeConX(end_base)));
- end = _gvn.transform(new AndXNode(end, MakeConX(~end_round)));
- mem = ClearArrayNode::clear_memory(control(), mem, dest,
- start_con, end, &_gvn);
- } else if (start_con < 0 && dest_size != top()) {
- // Non-constant start, pre-rounded end after the tail of the array.
- // This is almost certainly a "round-to-end" operation.
- Node* start = slice_idx;
- start = ConvI2X(start);
- if (scale != 0)
- start = _gvn.transform(new LShiftXNode( start, intcon(scale) ));
- start = _gvn.transform(new AddXNode(start, MakeConX(abase)));
- if ((bump_bit | clear_low) != 0) {
- int to_clear = (bump_bit | clear_low);
- // Align up mod 8, then store a jint zero unconditionally
- // just before the mod-8 boundary.
- if (((abase + bump_bit) & ~to_clear) - bump_bit
- < arrayOopDesc::length_offset_in_bytes() + BytesPerInt) {
- bump_bit = 0;
- assert((abase & to_clear) == 0, "array base must be long-aligned");
- } else {
- // Bump 'start' up to (or past) the next jint boundary:
- start = _gvn.transform(new AddXNode(start, MakeConX(bump_bit)));
- assert((abase & clear_low) == 0, "array base must be int-aligned");
- }
- // Round bumped 'start' down to jlong boundary in body of array.
- start = _gvn.transform(new AndXNode(start, MakeConX(~to_clear)));
- if (bump_bit != 0) {
- // Store a zero to the immediately preceding jint:
- Node* x1 = _gvn.transform(new AddXNode(start, MakeConX(-bump_bit)));
- Node* p1 = basic_plus_adr(dest, x1);
- mem = StoreNode::make(_gvn, control(), mem, p1, adr_type, intcon(0), T_INT, MemNode::unordered);
- mem = _gvn.transform(mem);
- }
- }
- Node* end = dest_size; // pre-rounded
- mem = ClearArrayNode::clear_memory(control(), mem, dest,
- start, end, &_gvn);
- } else {
- // Non-constant start, unrounded non-constant end.
- // (Nobody zeroes a random midsection of an array using this routine.)
- ShouldNotReachHere(); // fix caller
- }
-
- // Done.
- set_memory(mem, adr_type);
-}
-
-
-bool
-LibraryCallKit::generate_block_arraycopy(const TypePtr* adr_type,
- BasicType basic_elem_type,
- AllocateNode* alloc,
- Node* src, Node* src_offset,
- Node* dest, Node* dest_offset,
- Node* dest_size, bool dest_uninitialized) {
- // See if there is an advantage from block transfer.
- int scale = exact_log2(type2aelembytes(basic_elem_type));
- if (scale >= LogBytesPerLong)
- return false; // it is already a block transfer
-
- // Look at the alignment of the starting offsets.
- int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
-
- intptr_t src_off_con = (intptr_t) find_int_con(src_offset, -1);
- intptr_t dest_off_con = (intptr_t) find_int_con(dest_offset, -1);
- if (src_off_con < 0 || dest_off_con < 0)
- // At present, we can only understand constants.
- return false;
-
- intptr_t src_off = abase + (src_off_con << scale);
- intptr_t dest_off = abase + (dest_off_con << scale);
-
- if (((src_off | dest_off) & (BytesPerLong-1)) != 0) {
- // Non-aligned; too bad.
- // One more chance: Pick off an initial 32-bit word.
- // This is a common case, since abase can be odd mod 8.
- if (((src_off | dest_off) & (BytesPerLong-1)) == BytesPerInt &&
- ((src_off ^ dest_off) & (BytesPerLong-1)) == 0) {
- Node* sptr = basic_plus_adr(src, src_off);
- Node* dptr = basic_plus_adr(dest, dest_off);
- Node* sval = make_load(control(), sptr, TypeInt::INT, T_INT, adr_type, MemNode::unordered);
- store_to_memory(control(), dptr, sval, T_INT, adr_type, MemNode::unordered);
- src_off += BytesPerInt;
- dest_off += BytesPerInt;
- } else {
- return false;
- }
- }
- assert(src_off % BytesPerLong == 0, "");
- assert(dest_off % BytesPerLong == 0, "");
-
- // Do this copy by giant steps.
- Node* sptr = basic_plus_adr(src, src_off);
- Node* dptr = basic_plus_adr(dest, dest_off);
- Node* countx = dest_size;
- countx = _gvn.transform(new SubXNode(countx, MakeConX(dest_off)));
- countx = _gvn.transform(new URShiftXNode(countx, intcon(LogBytesPerLong)));
-
- bool disjoint_bases = true; // since alloc != NULL
- generate_unchecked_arraycopy(adr_type, T_LONG, disjoint_bases,
- sptr, NULL, dptr, NULL, countx, dest_uninitialized);
-
- return true;
-}
-
-
-// Helper function; generates code for the slow case.
-// We make a call to a runtime method which emulates the native method,
-// but without the native wrapper overhead.
-void
-LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type,
- Node* src, Node* src_offset,
- Node* dest, Node* dest_offset,
- Node* copy_length, bool dest_uninitialized) {
- assert(!dest_uninitialized, "Invariant");
- Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON,
- OptoRuntime::slow_arraycopy_Type(),
- OptoRuntime::slow_arraycopy_Java(),
- "slow_arraycopy", adr_type,
- src, src_offset, dest, dest_offset,
- copy_length);
-
- // Handle exceptions thrown by this fellow:
- make_slow_call_ex(call, env()->Throwable_klass(), false);
-}
-
-// Helper function; generates code for cases requiring runtime checks.
-Node*
-LibraryCallKit::generate_checkcast_arraycopy(const TypePtr* adr_type,
- Node* dest_elem_klass,
- Node* src, Node* src_offset,
- Node* dest, Node* dest_offset,
- Node* copy_length, bool dest_uninitialized) {
- if (stopped()) return NULL;
-
- address copyfunc_addr = StubRoutines::checkcast_arraycopy(dest_uninitialized);
- if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
- return NULL;
- }
-
- // Pick out the parameters required to perform a store-check
- // for the target array. This is an optimistic check. It will
- // look in each non-null element's class, at the desired klass's
- // super_check_offset, for the desired klass.
- int sco_offset = in_bytes(Klass::super_check_offset_offset());
- Node* p3 = basic_plus_adr(dest_elem_klass, sco_offset);
- Node* n3 = new LoadINode(NULL, memory(p3), p3, _gvn.type(p3)->is_ptr(), TypeInt::INT, MemNode::unordered);
- Node* check_offset = ConvI2X(_gvn.transform(n3));
- Node* check_value = dest_elem_klass;
-
- Node* src_start = array_element_address(src, src_offset, T_OBJECT);
- Node* dest_start = array_element_address(dest, dest_offset, T_OBJECT);
-
- // (We know the arrays are never conjoint, because their types differ.)
- Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
- OptoRuntime::checkcast_arraycopy_Type(),
- copyfunc_addr, "checkcast_arraycopy", adr_type,
- // five arguments, of which two are
- // intptr_t (jlong in LP64)
- src_start, dest_start,
- copy_length XTOP,
- check_offset XTOP,
- check_value);
-
- return _gvn.transform(new ProjNode(call, TypeFunc::Parms));
-}
-
-
-// Helper function; generates code for cases requiring runtime checks.
-Node*
-LibraryCallKit::generate_generic_arraycopy(const TypePtr* adr_type,
- Node* src, Node* src_offset,
- Node* dest, Node* dest_offset,
- Node* copy_length, bool dest_uninitialized) {
- assert(!dest_uninitialized, "Invariant");
- if (stopped()) return NULL;
- address copyfunc_addr = StubRoutines::generic_arraycopy();
- if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
- return NULL;
- }
-
- Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
- OptoRuntime::generic_arraycopy_Type(),
- copyfunc_addr, "generic_arraycopy", adr_type,
- src, src_offset, dest, dest_offset, copy_length);
-
- return _gvn.transform(new ProjNode(call, TypeFunc::Parms));
-}
-
-// Helper function; generates the fast out-of-line call to an arraycopy stub.
-void
-LibraryCallKit::generate_unchecked_arraycopy(const TypePtr* adr_type,
- BasicType basic_elem_type,
- bool disjoint_bases,
- Node* src, Node* src_offset,
- Node* dest, Node* dest_offset,
- Node* copy_length, bool dest_uninitialized) {
- if (stopped()) return; // nothing to do
-
- Node* src_start = src;
- Node* dest_start = dest;
- if (src_offset != NULL || dest_offset != NULL) {
- assert(src_offset != NULL && dest_offset != NULL, "");
- src_start = array_element_address(src, src_offset, basic_elem_type);
- dest_start = array_element_address(dest, dest_offset, basic_elem_type);
- }
-
- // Figure out which arraycopy runtime method to call.
- const char* copyfunc_name = "arraycopy";
- address copyfunc_addr =
- basictype2arraycopy(basic_elem_type, src_offset, dest_offset,
- disjoint_bases, copyfunc_name, dest_uninitialized);
-
- // Call it. Note that the count_ix value is not scaled to a byte-size.
- make_runtime_call(RC_LEAF|RC_NO_FP,
- OptoRuntime::fast_arraycopy_Type(),
- copyfunc_addr, copyfunc_name, adr_type,
- src_start, dest_start, copy_length XTOP);
-}
-
//-------------inline_encodeISOArray-----------------------------------
// encode char[] to byte[] in ISO_8859_1
bool LibraryCallKit::inline_encodeISOArray() {
--- a/hotspot/src/share/vm/opto/loopPredicate.cpp Fri Aug 08 11:36:48 2014 -0700
+++ b/hotspot/src/share/vm/opto/loopPredicate.cpp Mon Aug 11 14:12:51 2014 +0200
@@ -763,9 +763,7 @@
loop->dump_head();
}
#endif
- } else if ((cl != NULL) && (proj->_con == predicate_proj->_con) &&
- loop->is_range_check_if(iff, this, invar)) {
-
+ } else if (cl != NULL && loop->is_range_check_if(iff, this, invar)) {
// Range check for counted loops
const Node* cmp = bol->in(1)->as_Cmp();
Node* idx = cmp->in(1);
@@ -800,18 +798,31 @@
}
// Test the lower bound
- Node* lower_bound_bol = rc_predicate(loop, ctrl, scale, offset, init, limit, stride, rng, false);
+ BoolNode* lower_bound_bol = rc_predicate(loop, ctrl, scale, offset, init, limit, stride, rng, false);
+ // Negate test if necessary
+ bool negated = false;
+ if (proj->_con != predicate_proj->_con) {
+ lower_bound_bol = new BoolNode(lower_bound_bol->in(1), lower_bound_bol->_test.negate());
+ register_new_node(lower_bound_bol, ctrl);
+ negated = true;
+ }
IfNode* lower_bound_iff = lower_bound_proj->in(0)->as_If();
_igvn.hash_delete(lower_bound_iff);
lower_bound_iff->set_req(1, lower_bound_bol);
- if (TraceLoopPredicate) tty->print_cr("lower bound check if: %d", lower_bound_iff->_idx);
+ if (TraceLoopPredicate) tty->print_cr("lower bound check if: %s %d ", negated ? " negated" : "", lower_bound_iff->_idx);
// Test the upper bound
- Node* upper_bound_bol = rc_predicate(loop, lower_bound_proj, scale, offset, init, limit, stride, rng, true);
+ BoolNode* upper_bound_bol = rc_predicate(loop, lower_bound_proj, scale, offset, init, limit, stride, rng, true);
+ negated = false;
+ if (proj->_con != predicate_proj->_con) {
+ upper_bound_bol = new BoolNode(upper_bound_bol->in(1), upper_bound_bol->_test.negate());
+ register_new_node(upper_bound_bol, ctrl);
+ negated = true;
+ }
IfNode* upper_bound_iff = upper_bound_proj->in(0)->as_If();
_igvn.hash_delete(upper_bound_iff);
upper_bound_iff->set_req(1, upper_bound_bol);
- if (TraceLoopPredicate) tty->print_cr("upper bound check if: %d", lower_bound_iff->_idx);
+ if (TraceLoopPredicate) tty->print_cr("upper bound check if: %s %d ", negated ? " negated" : "", lower_bound_iff->_idx);
// Fall through into rest of the clean up code which will move
// any dependent nodes onto the upper bound test.
--- a/hotspot/src/share/vm/opto/macro.cpp Fri Aug 08 11:36:48 2014 -0700
+++ b/hotspot/src/share/vm/opto/macro.cpp Mon Aug 11 14:12:51 2014 +0200
@@ -2469,6 +2469,8 @@
assert(!n->as_AbstractLock()->is_eliminated(), "sanity");
_has_locks = true;
break;
+ case Node::Class_ArrayCopy:
+ break;
default:
assert(n->Opcode() == Op_LoopLimit ||
n->Opcode() == Op_Opaque1 ||
@@ -2544,6 +2546,25 @@
}
}
+ // expand arraycopy "macro" nodes first
+ // For ReduceBulkZeroing, we must first process all arraycopy nodes
+ // before the allocate nodes are expanded.
+ int macro_idx = C->macro_count() - 1;
+ while (macro_idx >= 0) {
+ Node * n = C->macro_node(macro_idx);
+ assert(n->is_macro(), "only macro nodes expected here");
+ if (_igvn.type(n) == Type::TOP || n->in(0)->is_top() ) {
+ // node is unreachable, so don't try to expand it
+ C->remove_macro_node(n);
+ } else if (n->is_ArrayCopy()){
+ int macro_count = C->macro_count();
+ expand_arraycopy_node(n->as_ArrayCopy());
+ assert(C->macro_count() < macro_count, "must have deleted a node from macro list");
+ }
+ if (C->failing()) return true;
+ macro_idx --;
+ }
+
// expand "macro" nodes
// nodes are removed from the macro list as they are processed
while (C->macro_count() > 0) {
--- a/hotspot/src/share/vm/opto/macro.hpp Fri Aug 08 11:36:48 2014 -0700
+++ b/hotspot/src/share/vm/opto/macro.hpp Mon Aug 11 14:12:51 2014 +0200
@@ -37,7 +37,7 @@
private:
PhaseIterGVN &_igvn;
- // Helper methods roughly modelled after GraphKit:
+ // Helper methods roughly modeled after GraphKit:
Node* top() const { return C->top(); }
Node* intcon(jint con) const { return _igvn.intcon(con); }
Node* longcon(jlong con) const { return _igvn.longcon(con); }
@@ -101,6 +101,86 @@
void expand_lock_node(LockNode *lock);
void expand_unlock_node(UnlockNode *unlock);
+ // More helper methods modeled after GraphKit for array copy
+ void insert_mem_bar(Node** ctrl, Node** mem, int opcode, Node* precedent = NULL);
+ Node* array_element_address(Node* ary, Node* idx, BasicType elembt);
+ Node* ConvI2L(Node* offset);
+ Node* make_leaf_call(Node* ctrl, Node* mem,
+ const TypeFunc* call_type, address call_addr,
+ const char* call_name,
+ const TypePtr* adr_type,
+ Node* parm0 = NULL, Node* parm1 = NULL,
+ Node* parm2 = NULL, Node* parm3 = NULL,
+ Node* parm4 = NULL, Node* parm5 = NULL,
+ Node* parm6 = NULL, Node* parm7 = NULL);
+
+ // helper methods modeled after LibraryCallKit for array copy
+ Node* generate_guard(Node** ctrl, Node* test, RegionNode* region, float true_prob);
+ Node* generate_slow_guard(Node** ctrl, Node* test, RegionNode* region);
+ void generate_negative_guard(Node** ctrl, Node* index, RegionNode* region);
+ void generate_limit_guard(Node** ctrl, Node* offset, Node* subseq_length, Node* array_length, RegionNode* region);
+
+ // More helper methods for array copy
+ Node* generate_nonpositive_guard(Node** ctrl, Node* index, bool never_negative);
+ void finish_arraycopy_call(Node* call, Node** ctrl, MergeMemNode** mem, const TypePtr* adr_type);
+ address basictype2arraycopy(BasicType t,
+ Node* src_offset,
+ Node* dest_offset,
+ bool disjoint_bases,
+ const char* &name,
+ bool dest_uninitialized);
+ Node* generate_arraycopy(ArrayCopyNode *ac,
+ AllocateArrayNode* alloc,
+ Node** ctrl, MergeMemNode* mem, Node** io,
+ const TypePtr* adr_type,
+ BasicType basic_elem_type,
+ Node* src, Node* src_offset,
+ Node* dest, Node* dest_offset,
+ Node* copy_length,
+ bool disjoint_bases = false,
+ bool length_never_negative = false,
+ RegionNode* slow_region = NULL);
+ void generate_clear_array(Node* ctrl, MergeMemNode* merge_mem,
+ const TypePtr* adr_type,
+ Node* dest,
+ BasicType basic_elem_type,
+ Node* slice_idx,
+ Node* slice_len,
+ Node* dest_size);
+ bool generate_block_arraycopy(Node** ctrl, MergeMemNode** mem, Node* io,
+ const TypePtr* adr_type,
+ BasicType basic_elem_type,
+ AllocateNode* alloc,
+ Node* src, Node* src_offset,
+ Node* dest, Node* dest_offset,
+ Node* dest_size, bool dest_uninitialized);
+ MergeMemNode* generate_slow_arraycopy(ArrayCopyNode *ac,
+ Node** ctrl, Node* mem, Node** io,
+ const TypePtr* adr_type,
+ Node* src, Node* src_offset,
+ Node* dest, Node* dest_offset,
+ Node* copy_length, bool dest_uninitialized);
+ Node* generate_checkcast_arraycopy(Node** ctrl, MergeMemNode** mem,
+ const TypePtr* adr_type,
+ Node* dest_elem_klass,
+ Node* src, Node* src_offset,
+ Node* dest, Node* dest_offset,
+ Node* copy_length, bool dest_uninitialized);
+ Node* generate_generic_arraycopy(Node** ctrl, MergeMemNode** mem,
+ const TypePtr* adr_type,
+ Node* src, Node* src_offset,
+ Node* dest, Node* dest_offset,
+ Node* copy_length, bool dest_uninitialized);
+ void generate_unchecked_arraycopy(Node** ctrl, MergeMemNode** mem,
+ const TypePtr* adr_type,
+ BasicType basic_elem_type,
+ bool disjoint_bases,
+ Node* src, Node* src_offset,
+ Node* dest, Node* dest_offset,
+ Node* copy_length, bool dest_uninitialized);
+
+ void expand_arraycopy_node(ArrayCopyNode *ac);
+
int replace_input(Node *use, Node *oldref, Node *newref);
void copy_call_debug_info(CallNode *oldcall, CallNode * newcall);
Node* opt_bits_test(Node* ctrl, Node* region, int edge, Node* word, int mask, int bits, bool return_fast_path = false);
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/hotspot/src/share/vm/opto/macroArrayCopy.cpp Mon Aug 11 14:12:51 2014 +0200
@@ -0,0 +1,1245 @@
+/*
+ * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "oops/objArrayKlass.hpp"
+#include "opto/convertnode.hpp"
+#include "opto/graphKit.hpp"
+#include "opto/macro.hpp"
+#include "opto/runtime.hpp"
+
+
+void PhaseMacroExpand::insert_mem_bar(Node** ctrl, Node** mem, int opcode, Node* precedent) {
+ MemBarNode* mb = MemBarNode::make(C, opcode, Compile::AliasIdxBot, precedent);
+ mb->init_req(TypeFunc::Control, *ctrl);
+ mb->init_req(TypeFunc::Memory, *mem);
+ transform_later(mb);
+ *ctrl = new ProjNode(mb,TypeFunc::Control);
+ transform_later(*ctrl);
+ Node* mem_proj = new ProjNode(mb,TypeFunc::Memory);
+ transform_later(mem_proj);
+ *mem = mem_proj;
+}
+
+Node* PhaseMacroExpand::array_element_address(Node* ary, Node* idx, BasicType elembt) {
+ uint shift = exact_log2(type2aelembytes(elembt));
+ uint header = arrayOopDesc::base_offset_in_bytes(elembt);
+ Node* base = basic_plus_adr(ary, header);
+#ifdef _LP64
+ // see comment in GraphKit::array_element_address
+ int index_max = max_jint - 1; // array size is max_jint, index is one less
+ const TypeLong* lidxtype = TypeLong::make(CONST64(0), index_max, Type::WidenMax);
+ idx = transform_later( new ConvI2LNode(idx, lidxtype) );
+#endif
+ Node* scale = new LShiftXNode(idx, intcon(shift));
+ transform_later(scale);
+ return basic_plus_adr(ary, base, scale);
+}
+
+Node* PhaseMacroExpand::ConvI2L(Node* offset) {
+ return transform_later(new ConvI2LNode(offset));
+}
+
+Node* PhaseMacroExpand::make_leaf_call(Node* ctrl, Node* mem,
+ const TypeFunc* call_type, address call_addr,
+ const char* call_name,
+ const TypePtr* adr_type,
+ Node* parm0, Node* parm1,
+ Node* parm2, Node* parm3,
+ Node* parm4, Node* parm5,
+ Node* parm6, Node* parm7) {
+ int size = call_type->domain()->cnt();
+ Node* call = new CallLeafNoFPNode(call_type, call_addr, call_name, adr_type);
+ call->init_req(TypeFunc::Control, ctrl);
+ call->init_req(TypeFunc::I_O , top());
+ call->init_req(TypeFunc::Memory , mem);
+ call->init_req(TypeFunc::ReturnAdr, top());
+ call->init_req(TypeFunc::FramePtr, top());
+
+ // Hook each parm in order. Stop looking at the first NULL.
+ if (parm0 != NULL) { call->init_req(TypeFunc::Parms+0, parm0);
+ if (parm1 != NULL) { call->init_req(TypeFunc::Parms+1, parm1);
+ if (parm2 != NULL) { call->init_req(TypeFunc::Parms+2, parm2);
+ if (parm3 != NULL) { call->init_req(TypeFunc::Parms+3, parm3);
+ if (parm4 != NULL) { call->init_req(TypeFunc::Parms+4, parm4);
+ if (parm5 != NULL) { call->init_req(TypeFunc::Parms+5, parm5);
+ if (parm6 != NULL) { call->init_req(TypeFunc::Parms+6, parm6);
+ if (parm7 != NULL) { call->init_req(TypeFunc::Parms+7, parm7);
+ /* close each nested if ===> */ } } } } } } } }
+ assert(call->in(call->req()-1) != NULL, "must initialize all parms");
+
+ return call;
+}
+
+
+//------------------------------generate_guard---------------------------
+// Helper function for generating guarded fast-slow graph structures.
+// The given 'test', if true, guards a slow path. If the test fails
+// then a fast path can be taken. (We generally hope it fails.)
+// In all cases, GraphKit::control() is updated to the fast path.
+// The returned value represents the control for the slow path.
+// The return value is never 'top'; it is either a valid control
+// or NULL if it is obvious that the slow path can never be taken.
+// Also, if region and the slow control are not NULL, the slow edge
+// is appended to the region.
+Node* PhaseMacroExpand::generate_guard(Node** ctrl, Node* test, RegionNode* region, float true_prob) {
+ if ((*ctrl)->is_top()) {
+ // Already short circuited.
+ return NULL;
+ }
+ // Build an if node and its projections.
+ // If test is true we take the slow path, which we assume is uncommon.
+ if (_igvn.type(test) == TypeInt::ZERO) {
+ // The slow branch is never taken. No need to build this guard.
+ return NULL;
+ }
+
+ IfNode* iff = new IfNode(*ctrl, test, true_prob, COUNT_UNKNOWN);
+ transform_later(iff);
+
+ Node* if_slow = new IfTrueNode(iff);
+ transform_later(if_slow);
+
+ if (region != NULL) {
+ region->add_req(if_slow);
+ }
+
+ Node* if_fast = new IfFalseNode(iff);
+ transform_later(if_fast);
+
+ *ctrl = if_fast;
+
+ return if_slow;
+}
+
+inline Node* PhaseMacroExpand::generate_slow_guard(Node** ctrl, Node* test, RegionNode* region) {
+ return generate_guard(ctrl, test, region, PROB_UNLIKELY_MAG(3));
+}
+
+void PhaseMacroExpand::generate_negative_guard(Node** ctrl, Node* index, RegionNode* region) {
+ if ((*ctrl)->is_top())
+ return; // already stopped
+ if (_igvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint]
+ return; // index is already adequately typed
+ Node* cmp_lt = new CmpINode(index, intcon(0));
+ transform_later(cmp_lt);
+ Node* bol_lt = new BoolNode(cmp_lt, BoolTest::lt);
+ transform_later(bol_lt);
+ generate_guard(ctrl, bol_lt, region, PROB_MIN);
+}
+
+void PhaseMacroExpand::generate_limit_guard(Node** ctrl, Node* offset, Node* subseq_length, Node* array_length, RegionNode* region) {
+ if ((*ctrl)->is_top())
+ return; // already stopped
+ bool zero_offset = _igvn.type(offset) == TypeInt::ZERO;
+ if (zero_offset && subseq_length->eqv_uncast(array_length))
+ return; // common case of whole-array copy
+ Node* last = subseq_length;
+ if (!zero_offset) { // last += offset
+ last = new AddINode(last, offset);
+ transform_later(last);
+ }
+ Node* cmp_lt = new CmpUNode(array_length, last);
+ transform_later(cmp_lt);
+ Node* bol_lt = new BoolNode(cmp_lt, BoolTest::lt);
+ transform_later(bol_lt);
+ generate_guard(ctrl, bol_lt, region, PROB_MIN);
+}
+
+Node* PhaseMacroExpand::generate_nonpositive_guard(Node** ctrl, Node* index, bool never_negative) {
+ if ((*ctrl)->is_top()) return NULL;
+
+ if (_igvn.type(index)->higher_equal(TypeInt::POS1)) // [1,maxint]
+ return NULL; // index is already adequately typed
+ Node* cmp_le = new CmpINode(index, intcon(0));
+ transform_later(cmp_le);
+ BoolTest::mask le_or_eq = (never_negative ? BoolTest::eq : BoolTest::le);
+ Node* bol_le = new BoolNode(cmp_le, le_or_eq);
+ transform_later(bol_le);
+ Node* is_notp = generate_guard(ctrl, bol_le, NULL, PROB_MIN);
+
+ return is_notp;
+}
+
+void PhaseMacroExpand::finish_arraycopy_call(Node* call, Node** ctrl, MergeMemNode** mem, const TypePtr* adr_type) {
+ transform_later(call);
+
+ *ctrl = new ProjNode(call,TypeFunc::Control);
+ transform_later(*ctrl);
+ Node* newmem = new ProjNode(call, TypeFunc::Memory);
+ transform_later(newmem);
+
+ uint alias_idx = C->get_alias_index(adr_type);
+ if (alias_idx != Compile::AliasIdxBot) {
+ *mem = MergeMemNode::make(*mem);
+ (*mem)->set_memory_at(alias_idx, newmem);
+ } else {
+ *mem = MergeMemNode::make(newmem);
+ }
+ transform_later(*mem);
+}
+
+address PhaseMacroExpand::basictype2arraycopy(BasicType t,
+ Node* src_offset,
+ Node* dest_offset,
+ bool disjoint_bases,
+ const char* &name,
+ bool dest_uninitialized) {
+ const TypeInt* src_offset_inttype = _igvn.find_int_type(src_offset);;
+ const TypeInt* dest_offset_inttype = _igvn.find_int_type(dest_offset);;
+
+ bool aligned = false;
+ bool disjoint = disjoint_bases;
+
+ // if the offsets are the same, we can treat the memory regions as
+ // disjoint, because either the memory regions are in different arrays,
+ // or they are identical (which we can treat as disjoint.) We can also
+ // treat a copy with a destination index less that the source index
+ // as disjoint since a low->high copy will work correctly in this case.
+ if (src_offset_inttype != NULL && src_offset_inttype->is_con() &&
+ dest_offset_inttype != NULL && dest_offset_inttype->is_con()) {
+ // both indices are constants
+ int s_offs = src_offset_inttype->get_con();
+ int d_offs = dest_offset_inttype->get_con();
+ int element_size = type2aelembytes(t);
+ aligned = ((arrayOopDesc::base_offset_in_bytes(t) + s_offs * element_size) % HeapWordSize == 0) &&
+ ((arrayOopDesc::base_offset_in_bytes(t) + d_offs * element_size) % HeapWordSize == 0);
+ if (s_offs >= d_offs) disjoint = true;
+ } else if (src_offset == dest_offset && src_offset != NULL) {
+ // This can occur if the offsets are identical non-constants.
+ disjoint = true;
+ }
+
+ return StubRoutines::select_arraycopy_function(t, aligned, disjoint, name, dest_uninitialized);
+}
+
+#define COMMA ,
+#define XTOP LP64_ONLY(COMMA top())
+
+// Generate an optimized call to arraycopy.
+// Caller must guard against non-arrays.
+// Caller must determine a common array basic-type for both arrays.
+// Caller must validate offsets against array bounds.
+// The slow_region has already collected guard failure paths
+// (such as out of bounds length or non-conformable array types).
+// The generated code has this shape, in general:
+//
+// if (length == 0) return // via zero_path
+// slowval = -1
+// if (types unknown) {
+// slowval = call generic copy loop
+// if (slowval == 0) return // via checked_path
+// } else if (indexes in bounds) {
+// if ((is object array) && !(array type check)) {
+// slowval = call checked copy loop
+// if (slowval == 0) return // via checked_path
+// } else {
+// call bulk copy loop
+// return // via fast_path
+// }
+// }
+// // adjust params for remaining work:
+// if (slowval != -1) {
+// n = -1^slowval; src_offset += n; dest_offset += n; length -= n
+// }
+// slow_region:
+// call slow arraycopy(src, src_offset, dest, dest_offset, length)
+// return // via slow_call_path
+//
+// This routine is used from several intrinsics: System.arraycopy,
+// Object.clone (the array subcase), and Arrays.copyOf[Range].
+//
+Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* alloc,
+ Node** ctrl, MergeMemNode* mem, Node** io,
+ const TypePtr* adr_type,
+ BasicType basic_elem_type,
+ Node* src, Node* src_offset,
+ Node* dest, Node* dest_offset,
+ Node* copy_length,
+ bool disjoint_bases,
+ bool length_never_negative,
+ RegionNode* slow_region) {
+ if (slow_region == NULL) {
+ slow_region = new RegionNode(1);
+ transform_later(slow_region);
+ }
+
+ Node* original_dest = dest;
+ bool dest_uninitialized = false;
+
+ // See if this is the initialization of a newly-allocated array.
+ // If so, we will take responsibility here for initializing it to zero.
+ // (Note: Because tightly_coupled_allocation performs checks on the
+ // out-edges of the dest, we need to avoid making derived pointers
+ // from it until we have checked its uses.)
+ if (ReduceBulkZeroing
+ && !ZeroTLAB // pointless if already zeroed
+ && basic_elem_type != T_CONFLICT // avoid corner case
+ && !src->eqv_uncast(dest)
+ && alloc != NULL
+ && _igvn.find_int_con(alloc->in(AllocateNode::ALength), 1) > 0
+ && alloc->maybe_set_complete(&_igvn)) {
+ // "You break it, you buy it."
+ InitializeNode* init = alloc->initialization();
+ assert(init->is_complete(), "we just did this");
+ init->set_complete_with_arraycopy();
+ assert(dest->is_CheckCastPP(), "sanity");
+ assert(dest->in(0)->in(0) == init, "dest pinned");
+ adr_type = TypeRawPtr::BOTTOM; // all initializations are into raw memory
+ // From this point on, every exit path is responsible for
+ // initializing any non-copied parts of the object to zero.
+ // Also, if this flag is set we make sure that arraycopy interacts properly
+ // with G1, eliding pre-barriers. See CR 6627983.
+ dest_uninitialized = true;
+ } else {
+ // No zeroing elimination here.
+ alloc = NULL;
+ //original_dest = dest;
+ //dest_uninitialized = false;
+ }
+
+ uint alias_idx = C->get_alias_index(adr_type);
+
+ // Results are placed here:
+ enum { fast_path = 1, // normal void-returning assembly stub
+ checked_path = 2, // special assembly stub with cleanup
+ slow_call_path = 3, // something went wrong; call the VM
+ zero_path = 4, // bypass when length of copy is zero
+ bcopy_path = 5, // copy primitive array by 64-bit blocks
+ PATH_LIMIT = 6
+ };
+ RegionNode* result_region = new RegionNode(PATH_LIMIT);
+ PhiNode* result_i_o = new PhiNode(result_region, Type::ABIO);
+ PhiNode* result_memory = new PhiNode(result_region, Type::MEMORY, adr_type);
+ assert(adr_type != TypePtr::BOTTOM, "must be RawMem or a T[] slice");
+ transform_later(result_region);
+ transform_later(result_i_o);
+ transform_later(result_memory);
+
+ // The slow_control path:
+ Node* slow_control;
+ Node* slow_i_o = *io;
+ Node* slow_mem = mem->memory_at(alias_idx);
+ DEBUG_ONLY(slow_control = (Node*) badAddress);
+
+ // Checked control path:
+ Node* checked_control = top();
+ Node* checked_mem = NULL;
+ Node* checked_i_o = NULL;
+ Node* checked_value = NULL;
+
+ if (basic_elem_type == T_CONFLICT) {
+ assert(!dest_uninitialized, "");
+ Node* cv = generate_generic_arraycopy(ctrl, &mem,
+ adr_type,
+ src, src_offset, dest, dest_offset,
+ copy_length, dest_uninitialized);
+ if (cv == NULL) cv = intcon(-1); // failure (no stub available)
+ checked_control = *ctrl;
+ checked_i_o = *io;
+ checked_mem = mem->memory_at(alias_idx);
+ checked_value = cv;
+ *ctrl = top();
+ }
+
+ Node* not_pos = generate_nonpositive_guard(ctrl, copy_length, length_never_negative);
+ if (not_pos != NULL) {
+ Node* local_ctrl = not_pos, *local_io = *io;
+ MergeMemNode* local_mem = MergeMemNode::make(mem);
+ transform_later(local_mem);
+
+ // (6) length must not be negative.
+ if (!length_never_negative) {
+ generate_negative_guard(&local_ctrl, copy_length, slow_region);
+ }
+
+ // copy_length is 0.
+ if (dest_uninitialized) {
+ assert(!local_ctrl->is_top(), "no ctrl?");
+ Node* dest_length = alloc->in(AllocateNode::ALength);
+ if (copy_length->eqv_uncast(dest_length)
+ || _igvn.find_int_con(dest_length, 1) <= 0) {
+ // There is no zeroing to do. No need for a secondary raw memory barrier.
+ } else {
+ // Clear the whole thing since there are no source elements to copy.
+ generate_clear_array(local_ctrl, local_mem,
+ adr_type, dest, basic_elem_type,
+ intcon(0), NULL,
+ alloc->in(AllocateNode::AllocSize));
+ // Use a secondary InitializeNode as raw memory barrier.
+ // Currently it is needed only on this path since other
+ // paths have stub or runtime calls as raw memory barriers.
+ MemBarNode* mb = MemBarNode::make(C, Op_Initialize,
+ Compile::AliasIdxRaw,
+ top());
+ transform_later(mb);
+ mb->set_req(TypeFunc::Control,local_ctrl);
+ mb->set_req(TypeFunc::Memory, local_mem->memory_at(Compile::AliasIdxRaw));
+ local_ctrl = transform_later(new ProjNode(mb, TypeFunc::Control));
+ local_mem->set_memory_at(Compile::AliasIdxRaw, transform_later(new ProjNode(mb, TypeFunc::Memory)));
+
+ InitializeNode* init = mb->as_Initialize();
+ init->set_complete(&_igvn); // (there is no corresponding AllocateNode)
+ }
+ }
+
+ // Present the results of the fast call.
+ result_region->init_req(zero_path, local_ctrl);
+ result_i_o ->init_req(zero_path, local_io);
+ result_memory->init_req(zero_path, local_mem->memory_at(alias_idx));
+ }
+
+ if (!(*ctrl)->is_top() && dest_uninitialized) {
+ // We have to initialize the *uncopied* part of the array to zero.
+ // The copy destination is the slice dest[off..off+len]. The other slices
+ // are dest_head = dest[0..off] and dest_tail = dest[off+len..dest.length].
+ Node* dest_size = alloc->in(AllocateNode::AllocSize);
+ Node* dest_length = alloc->in(AllocateNode::ALength);
+ Node* dest_tail = transform_later( new AddINode(dest_offset, copy_length));
+
+ // If there is a head section that needs zeroing, do it now.
+ if (_igvn.find_int_con(dest_offset, -1) != 0) {
+ generate_clear_array(*ctrl, mem,
+ adr_type, dest, basic_elem_type,
+ intcon(0), dest_offset,
+ NULL);
+ }
+
+ // Next, perform a dynamic check on the tail length.
+ // It is often zero, and we can win big if we prove this.
+ // There are two wins: Avoid generating the ClearArray
+ // with its attendant messy index arithmetic, and upgrade
+ // the copy to a more hardware-friendly word size of 64 bits.
+ Node* tail_ctl = NULL;
+ if (!(*ctrl)->is_top() && !dest_tail->eqv_uncast(dest_length)) {
+ Node* cmp_lt = transform_later( new CmpINode(dest_tail, dest_length) );
+ Node* bol_lt = transform_later( new BoolNode(cmp_lt, BoolTest::lt) );
+ tail_ctl = generate_slow_guard(ctrl, bol_lt, NULL);
+ assert(tail_ctl != NULL || !(*ctrl)->is_top(), "must be an outcome");
+ }
+
+ // At this point, let's assume there is no tail.
+ if (!(*ctrl)->is_top() && alloc != NULL && basic_elem_type != T_OBJECT) {
+ // There is no tail. Try an upgrade to a 64-bit copy.
+ bool didit = false;
+ {
+ Node* local_ctrl = *ctrl, *local_io = *io;
+ MergeMemNode* local_mem = MergeMemNode::make(mem);
+ transform_later(local_mem);
+
+ didit = generate_block_arraycopy(&local_ctrl, &local_mem, local_io,
+ adr_type, basic_elem_type, alloc,
+ src, src_offset, dest, dest_offset,
+ dest_size, dest_uninitialized);
+ if (didit) {
+ // Present the results of the block-copying fast call.
+ result_region->init_req(bcopy_path, local_ctrl);
+ result_i_o ->init_req(bcopy_path, local_io);
+ result_memory->init_req(bcopy_path, local_mem->memory_at(alias_idx));
+ }
+ }
+ if (didit) {
+ *ctrl = top(); // no regular fast path
+ }
+ }
+
+ // Clear the tail, if any.
+ if (tail_ctl != NULL) {
+ Node* notail_ctl = (*ctrl)->is_top() ? NULL : *ctrl;
+ *ctrl = tail_ctl;
+ if (notail_ctl == NULL) {
+ generate_clear_array(*ctrl, mem,
+ adr_type, dest, basic_elem_type,
+ dest_tail, NULL,
+ dest_size);
+ } else {
+ // Make a local merge.
+ Node* done_ctl = transform_later(new RegionNode(3));
+ Node* done_mem = transform_later(new PhiNode(done_ctl, Type::MEMORY, adr_type));
+ done_ctl->init_req(1, notail_ctl);
+ done_mem->init_req(1, mem->memory_at(alias_idx));
+ generate_clear_array(*ctrl, mem,
+ adr_type, dest, basic_elem_type,
+ dest_tail, NULL,
+ dest_size);
+ done_ctl->init_req(2, *ctrl);
+ done_mem->init_req(2, mem->memory_at(alias_idx));
+ *ctrl = done_ctl;
+ mem->set_memory_at(alias_idx, done_mem);
+ }
+ }
+ }
+
+ BasicType copy_type = basic_elem_type;
+ assert(basic_elem_type != T_ARRAY, "caller must fix this");
+ if (!(*ctrl)->is_top() && copy_type == T_OBJECT) {
+ // If src and dest have compatible element types, we can copy bits.
+ // Types S[] and D[] are compatible if D is a supertype of S.
+ //
+ // If they are not, we will use checked_oop_disjoint_arraycopy,
+ // which performs a fast optimistic per-oop check, and backs off
+ // further to JVM_ArrayCopy on the first per-oop check that fails.
+ // (Actually, we don't move raw bits only; the GC requires card marks.)
+
+ // Get the klass* for both src and dest
+ Node* k_adr = new AddPNode(src, src, MakeConX(oopDesc::klass_offset_in_bytes()));
+ transform_later(k_adr);
+ Node* src_klass = LoadKlassNode::make(_igvn, C->immutable_memory(), k_adr, TypeInstPtr::KLASS);
+ transform_later(src_klass);
+ k_adr = new AddPNode(dest, dest, MakeConX(oopDesc::klass_offset_in_bytes()));
+ transform_later(k_adr);
+ Node* dest_klass = LoadKlassNode::make(_igvn, C->immutable_memory(), k_adr, TypeInstPtr::KLASS);
+ transform_later(dest_klass);
+
+ // Generate the subtype check.
+ // This might fold up statically, or then again it might not.
+ //
+ // Non-static example: Copying List<String>.elements to a new String[].
+ // The backing store for a List<String> is always an Object[],
+ // but its elements are always type String, if the generic types
+ // are correct at the source level.
+ //
+ // Test S[] against D[], not S against D, because (probably)
+ // the secondary supertype cache is less busy for S[] than S.
+ // This usually only matters when D is an interface.
+ Node* not_subtype_ctrl = ac->is_arraycopy_notest() ? top() : Phase::gen_subtype_check(src_klass, dest_klass, ctrl, mem, &_igvn);
+ // Plug failing path into checked_oop_disjoint_arraycopy
+ if (not_subtype_ctrl != top()) {
+ Node* local_ctrl = not_subtype_ctrl;
+ MergeMemNode* local_mem = MergeMemNode::make(mem);
+ transform_later(local_mem);
+
+ // (At this point we can assume disjoint_bases, since types differ.)
+ int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
+ Node* p1 = basic_plus_adr(dest_klass, ek_offset);
+ Node* n1 = LoadKlassNode::make(_igvn, C->immutable_memory(), p1, TypeRawPtr::BOTTOM);
+ Node* dest_elem_klass = transform_later(n1);
+ Node* cv = generate_checkcast_arraycopy(&local_ctrl, &local_mem,
+ adr_type,
+ dest_elem_klass,
+ src, src_offset, dest, dest_offset,
+ ConvI2X(copy_length), dest_uninitialized);
+ if (cv == NULL) cv = intcon(-1); // failure (no stub available)
+ checked_control = local_ctrl;
+ checked_i_o = *io;
+ checked_mem = local_mem->memory_at(alias_idx);
+ checked_value = cv;
+ }
+ // At this point we know we do not need type checks on oop stores.
+
+ // Let's see if we need card marks:
+ if (alloc != NULL && GraphKit::use_ReduceInitialCardMarks()) {
+ // If we do not need card marks, copy using the jint or jlong stub.
+ copy_type = LP64_ONLY(UseCompressedOops ? T_INT : T_LONG) NOT_LP64(T_INT);
+ assert(type2aelembytes(basic_elem_type) == type2aelembytes(copy_type),
+ "sizes agree");
+ }
+ }
+
+ if (!(*ctrl)->is_top()) {
+ // Generate the fast path, if possible.
+ Node* local_ctrl = *ctrl;
+ MergeMemNode* local_mem = MergeMemNode::make(mem);
+ transform_later(local_mem);
+
+ generate_unchecked_arraycopy(&local_ctrl, &local_mem,
+ adr_type, copy_type, disjoint_bases,
+ src, src_offset, dest, dest_offset,
+ ConvI2X(copy_length), dest_uninitialized);
+
+ // Present the results of the fast call.
+ result_region->init_req(fast_path, local_ctrl);
+ result_i_o ->init_req(fast_path, *io);
+ result_memory->init_req(fast_path, local_mem->memory_at(alias_idx));
+ }
+
+ // Here are all the slow paths up to this point, in one bundle:
+ assert(slow_region != NULL, "allocated on entry");
+ slow_control = slow_region;
+ DEBUG_ONLY(slow_region = (RegionNode*)badAddress);
+
+ *ctrl = checked_control;
+ if (!(*ctrl)->is_top()) {
+ // Clean up after the checked call.
+ // The returned value is either 0 or -1^K,
+ // where K = number of partially transferred array elements.
+ Node* cmp = new CmpINode(checked_value, intcon(0));
+ transform_later(cmp);
+ Node* bol = new BoolNode(cmp, BoolTest::eq);
+ transform_later(bol);
+ IfNode* iff = new IfNode(*ctrl, bol, PROB_MAX, COUNT_UNKNOWN);
+ transform_later(iff);
+
+ // If it is 0, we are done, so transfer to the end.
+ Node* checks_done = new IfTrueNode(iff);
+ transform_later(checks_done);
+ result_region->init_req(checked_path, checks_done);
+ result_i_o ->init_req(checked_path, checked_i_o);
+ result_memory->init_req(checked_path, checked_mem);
+
+ // If it is not zero, merge into the slow call.
+ *ctrl = new IfFalseNode(iff);
+ transform_later(*ctrl);
+ RegionNode* slow_reg2 = new RegionNode(3);
+ PhiNode* slow_i_o2 = new PhiNode(slow_reg2, Type::ABIO);
+ PhiNode* slow_mem2 = new PhiNode(slow_reg2, Type::MEMORY, adr_type);
+ transform_later(slow_reg2);
+ transform_later(slow_i_o2);
+ transform_later(slow_mem2);
+ slow_reg2 ->init_req(1, slow_control);
+ slow_i_o2 ->init_req(1, slow_i_o);
+ slow_mem2 ->init_req(1, slow_mem);
+ slow_reg2 ->init_req(2, *ctrl);
+ slow_i_o2 ->init_req(2, checked_i_o);
+ slow_mem2 ->init_req(2, checked_mem);
+
+ slow_control = slow_reg2;
+ slow_i_o = slow_i_o2;
+ slow_mem = slow_mem2;
+
+ if (alloc != NULL) {
+ // We'll restart from the very beginning, after zeroing the whole thing.
+ // This can cause double writes, but that's OK since dest is brand new.
+ // So we ignore the low 31 bits of the value returned from the stub.
+ } else {
+ // We must continue the copy exactly where it failed, or else
+ // another thread might see the wrong number of writes to dest.
+ Node* checked_offset = new XorINode(checked_value, intcon(-1));
+ Node* slow_offset = new PhiNode(slow_reg2, TypeInt::INT);
+ transform_later(checked_offset);
+ transform_later(slow_offset);
+ slow_offset->init_req(1, intcon(0));
+ slow_offset->init_req(2, checked_offset);
+
+ // Adjust the arguments by the conditionally incoming offset.
+ Node* src_off_plus = new AddINode(src_offset, slow_offset);
+ transform_later(src_off_plus);
+ Node* dest_off_plus = new AddINode(dest_offset, slow_offset);
+ transform_later(dest_off_plus);
+ Node* length_minus = new SubINode(copy_length, slow_offset);
+ transform_later(length_minus);
+
+ // Tweak the node variables to adjust the code produced below:
+ src_offset = src_off_plus;
+ dest_offset = dest_off_plus;
+ copy_length = length_minus;
+ }
+ }
+ *ctrl = slow_control;
+ if (!(*ctrl)->is_top()) {
+ Node* local_ctrl = *ctrl, *local_io = slow_i_o;
+ MergeMemNode* local_mem = MergeMemNode::make(mem);
+ transform_later(local_mem);
+
+ // Generate the slow path, if needed.
+ local_mem->set_memory_at(alias_idx, slow_mem);
+
+ if (dest_uninitialized) {
+ generate_clear_array(local_ctrl, local_mem,
+ adr_type, dest, basic_elem_type,
+ intcon(0), NULL,
+ alloc->in(AllocateNode::AllocSize));
+ }
+
+ local_mem = generate_slow_arraycopy(ac,
+ &local_ctrl, local_mem, &local_io,
+ adr_type,
+ src, src_offset, dest, dest_offset,
+ copy_length, /*dest_uninitialized*/false);
+
+ result_region->init_req(slow_call_path, local_ctrl);
+ result_i_o ->init_req(slow_call_path, local_io);
+ result_memory->init_req(slow_call_path, local_mem->memory_at(alias_idx));
+ } else {
+ ShouldNotReachHere(); // no call to generate_slow_arraycopy:
+ // projections were not extracted
+ }
+
+ // Remove unused edges.
+ for (uint i = 1; i < result_region->req(); i++) {
+ if (result_region->in(i) == NULL) {
+ result_region->init_req(i, top());
+ }
+ }
+
+ // Finished; return the combined state.
+ *ctrl = result_region;
+ *io = result_i_o;
+ mem->set_memory_at(alias_idx, result_memory);
+
+ // mem no longer guaranteed to stay a MergeMemNode
+ Node* out_mem = mem;
+ DEBUG_ONLY(mem = NULL);
+
+ // The memory edges above are precise in order to model effects around
+ // array copies accurately to allow value numbering of field loads around
+ // arraycopy. Such field loads, both before and after, are common in Java
+ // collections and similar classes involving header/array data structures.
+ //
+ // But with low number of register or when some registers are used or killed
+ // by arraycopy calls it causes registers spilling on stack. See 6544710.
+ // The next memory barrier is added to avoid it. If the arraycopy can be
+ // optimized away (which it can, sometimes) then we can manually remove
+ // the membar also.
+ //
+ // Do not let reads from the cloned object float above the arraycopy.
+ if (alloc != NULL && !alloc->initialization()->does_not_escape()) {
+ // Do not let stores that initialize this object be reordered with
+ // a subsequent store that would make this object accessible by
+ // other threads.
+ insert_mem_bar(ctrl, &out_mem, Op_MemBarStoreStore);
+ } else if (InsertMemBarAfterArraycopy) {
+ insert_mem_bar(ctrl, &out_mem, Op_MemBarCPUOrder);
+ }
+
+ _igvn.replace_node(_memproj_fallthrough, out_mem);
+ _igvn.replace_node(_ioproj_fallthrough, *io);
+ _igvn.replace_node(_fallthroughcatchproj, *ctrl);
+
+ return out_mem;
+}
+
+// Helper for initialization of arrays, creating a ClearArray.
+// It writes zero bits in [start..end), within the body of an array object.
+// The memory effects are all chained onto the 'adr_type' alias category.
+//
+// Since the object is otherwise uninitialized, we are free
+// to put a little "slop" around the edges of the cleared area,
+// as long as it does not go back into the array's header,
+// or beyond the array end within the heap.
+//
+// The lower edge can be rounded down to the nearest jint and the
+// upper edge can be rounded up to the nearest MinObjAlignmentInBytes.
+//
+// Arguments:
+// adr_type memory slice where writes are generated
+// dest oop of the destination array
+// basic_elem_type element type of the destination
+// slice_idx array index of first element to store
+// slice_len number of elements to store (or NULL)
+// dest_size total size in bytes of the array object
+//
+// Exactly one of slice_len or dest_size must be non-NULL.
+// If dest_size is non-NULL, zeroing extends to the end of the object.
+// If slice_len is non-NULL, the slice_idx value must be a constant.
+void PhaseMacroExpand::generate_clear_array(Node* ctrl, MergeMemNode* merge_mem,
+ const TypePtr* adr_type,
+ Node* dest,
+ BasicType basic_elem_type,
+ Node* slice_idx,
+ Node* slice_len,
+ Node* dest_size) {
+ // one or the other but not both of slice_len and dest_size:
+ assert((slice_len != NULL? 1: 0) + (dest_size != NULL? 1: 0) == 1, "");
+ if (slice_len == NULL) slice_len = top();
+ if (dest_size == NULL) dest_size = top();
+
+ uint alias_idx = C->get_alias_index(adr_type);
+
+ // operate on this memory slice:
+ Node* mem = merge_mem->memory_at(alias_idx); // memory slice to operate on
+
+ // scaling and rounding of indexes:
+ int scale = exact_log2(type2aelembytes(basic_elem_type));
+ int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
+ int clear_low = (-1 << scale) & (BytesPerInt - 1);
+ int bump_bit = (-1 << scale) & BytesPerInt;
+
+ // determine constant starts and ends
+ const intptr_t BIG_NEG = -128;
+ assert(BIG_NEG + 2*abase < 0, "neg enough");
+ intptr_t slice_idx_con = (intptr_t) _igvn.find_int_con(slice_idx, BIG_NEG);
+ intptr_t slice_len_con = (intptr_t) _igvn.find_int_con(slice_len, BIG_NEG);
+ if (slice_len_con == 0) {
+ return; // nothing to do here
+ }
+ intptr_t start_con = (abase + (slice_idx_con << scale)) & ~clear_low;
+ intptr_t end_con = _igvn.find_intptr_t_con(dest_size, -1);
+ if (slice_idx_con >= 0 && slice_len_con >= 0) {
+ assert(end_con < 0, "not two cons");
+ end_con = round_to(abase + ((slice_idx_con + slice_len_con) << scale),
+ BytesPerLong);
+ }
+
+ if (start_con >= 0 && end_con >= 0) {
+ // Constant start and end. Simple.
+ mem = ClearArrayNode::clear_memory(ctrl, mem, dest,
+ start_con, end_con, &_igvn);
+ } else if (start_con >= 0 && dest_size != top()) {
+ // Constant start, pre-rounded end after the tail of the array.
+ Node* end = dest_size;
+ mem = ClearArrayNode::clear_memory(ctrl, mem, dest,
+ start_con, end, &_igvn);
+ } else if (start_con >= 0 && slice_len != top()) {
+ // Constant start, non-constant end. End needs rounding up.
+ // End offset = round_up(abase + ((slice_idx_con + slice_len) << scale), 8)
+ intptr_t end_base = abase + (slice_idx_con << scale);
+ int end_round = (-1 << scale) & (BytesPerLong - 1);
+ Node* end = ConvI2X(slice_len);
+ if (scale != 0)
+ end = transform_later(new LShiftXNode(end, intcon(scale) ));
+ end_base += end_round;
+ end = transform_later(new AddXNode(end, MakeConX(end_base)) );
+ end = transform_later(new AndXNode(end, MakeConX(~end_round)) );
+ mem = ClearArrayNode::clear_memory(ctrl, mem, dest,
+ start_con, end, &_igvn);
+ } else if (start_con < 0 && dest_size != top()) {
+ // Non-constant start, pre-rounded end after the tail of the array.
+ // This is almost certainly a "round-to-end" operation.
+ Node* start = slice_idx;
+ start = ConvI2X(start);
+ if (scale != 0)
+ start = transform_later(new LShiftXNode( start, intcon(scale) ));
+ start = transform_later(new AddXNode(start, MakeConX(abase)) );
+ if ((bump_bit | clear_low) != 0) {
+ int to_clear = (bump_bit | clear_low);
+ // Align up mod 8, then store a jint zero unconditionally
+ // just before the mod-8 boundary.
+ if (((abase + bump_bit) & ~to_clear) - bump_bit
+ < arrayOopDesc::length_offset_in_bytes() + BytesPerInt) {
+ bump_bit = 0;
+ assert((abase & to_clear) == 0, "array base must be long-aligned");
+ } else {
+ // Bump 'start' up to (or past) the next jint boundary:
+ start = transform_later( new AddXNode(start, MakeConX(bump_bit)) );
+ assert((abase & clear_low) == 0, "array base must be int-aligned");
+ }
+ // Round bumped 'start' down to jlong boundary in body of array.
+ start = transform_later(new AndXNode(start, MakeConX(~to_clear)) );
+ if (bump_bit != 0) {
+ // Store a zero to the immediately preceding jint:
+ Node* x1 = transform_later(new AddXNode(start, MakeConX(-bump_bit)) );
+ Node* p1 = basic_plus_adr(dest, x1);
+ mem = StoreNode::make(_igvn, ctrl, mem, p1, adr_type, intcon(0), T_INT, MemNode::unordered);
+ mem = transform_later(mem);
+ }
+ }
+ Node* end = dest_size; // pre-rounded
+ mem = ClearArrayNode::clear_memory(ctrl, mem, dest,
+ start, end, &_igvn);
+ } else {
+ // Non-constant start, unrounded non-constant end.
+ // (Nobody zeroes a random midsection of an array using this routine.)
+ ShouldNotReachHere(); // fix caller
+ }
+
+ // Done.
+ merge_mem->set_memory_at(alias_idx, mem);
+}
+
+bool PhaseMacroExpand::generate_block_arraycopy(Node** ctrl, MergeMemNode** mem, Node* io,
+ const TypePtr* adr_type,
+ BasicType basic_elem_type,
+ AllocateNode* alloc,
+ Node* src, Node* src_offset,
+ Node* dest, Node* dest_offset,
+ Node* dest_size, bool dest_uninitialized) {
+ // See if there is an advantage from block transfer.
+ int scale = exact_log2(type2aelembytes(basic_elem_type));
+ if (scale >= LogBytesPerLong)
+ return false; // it is already a block transfer
+
+ // Look at the alignment of the starting offsets.
+ int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
+
+ intptr_t src_off_con = (intptr_t) _igvn.find_int_con(src_offset, -1);
+ intptr_t dest_off_con = (intptr_t) _igvn.find_int_con(dest_offset, -1);
+ if (src_off_con < 0 || dest_off_con < 0) {
+ // At present, we can only understand constants.
+ return false;
+ }
+
+ intptr_t src_off = abase + (src_off_con << scale);
+ intptr_t dest_off = abase + (dest_off_con << scale);
+
+ if (((src_off | dest_off) & (BytesPerLong-1)) != 0) {
+ // Non-aligned; too bad.
+ // One more chance: Pick off an initial 32-bit word.
+ // This is a common case, since abase can be odd mod 8.
+ if (((src_off | dest_off) & (BytesPerLong-1)) == BytesPerInt &&
+ ((src_off ^ dest_off) & (BytesPerLong-1)) == 0) {
+ Node* sptr = basic_plus_adr(src, src_off);
+ Node* dptr = basic_plus_adr(dest, dest_off);
+ uint alias_idx = C->get_alias_index(adr_type);
+ Node* sval = transform_later(LoadNode::make(_igvn, *ctrl, (*mem)->memory_at(alias_idx), sptr, adr_type, TypeInt::INT, T_INT, MemNode::unordered));
+ Node* st = transform_later(StoreNode::make(_igvn, *ctrl, (*mem)->memory_at(alias_idx), dptr, adr_type, sval, T_INT, MemNode::unordered));
+ (*mem)->set_memory_at(alias_idx, st);
+ src_off += BytesPerInt;
+ dest_off += BytesPerInt;
+ } else {
+ return false;
+ }
+ }
+ assert(src_off % BytesPerLong == 0, "");
+ assert(dest_off % BytesPerLong == 0, "");
+
+ // Do this copy by giant steps.
+ Node* sptr = basic_plus_adr(src, src_off);
+ Node* dptr = basic_plus_adr(dest, dest_off);
+ Node* countx = dest_size;
+ countx = transform_later(new SubXNode(countx, MakeConX(dest_off)));
+ countx = transform_later(new URShiftXNode(countx, intcon(LogBytesPerLong)));
+
+ bool disjoint_bases = true; // since alloc != NULL
+ generate_unchecked_arraycopy(ctrl, mem,
+ adr_type, T_LONG, disjoint_bases,
+ sptr, NULL, dptr, NULL, countx, dest_uninitialized);
+
+ return true;
+}
+
+// Helper function; generates code for the slow case.
+// We make a call to a runtime method which emulates the native method,
+// but without the native wrapper overhead.
+MergeMemNode* PhaseMacroExpand::generate_slow_arraycopy(ArrayCopyNode *ac,
+ Node** ctrl, Node* mem, Node** io,
+ const TypePtr* adr_type,
+ Node* src, Node* src_offset,
+ Node* dest, Node* dest_offset,
+ Node* copy_length, bool dest_uninitialized) {
+ assert(!dest_uninitialized, "Invariant");
+
+ const TypeFunc* call_type = OptoRuntime::slow_arraycopy_Type();
+ CallNode* call = new CallStaticJavaNode(call_type, OptoRuntime::slow_arraycopy_Java(),
+ "slow_arraycopy",
+ ac->jvms()->bci(), TypePtr::BOTTOM);
+
+ call->init_req(TypeFunc::Control, *ctrl);
+ call->init_req(TypeFunc::I_O , *io);
+ call->init_req(TypeFunc::Memory , mem);
+ call->init_req(TypeFunc::ReturnAdr, top());
+ call->init_req(TypeFunc::FramePtr, top());
+ call->init_req(TypeFunc::Parms+0, src);
+ call->init_req(TypeFunc::Parms+1, src_offset);
+ call->init_req(TypeFunc::Parms+2, dest);
+ call->init_req(TypeFunc::Parms+3, dest_offset);
+ call->init_req(TypeFunc::Parms+4, copy_length);
+ copy_call_debug_info(ac, call);
+
+ call->set_cnt(PROB_UNLIKELY_MAG(4)); // Same effect as RC_UNCOMMON.
+ _igvn.replace_node(ac, call);
+ transform_later(call);
+
+ extract_call_projections(call);
+ *ctrl = _fallthroughcatchproj->clone();
+ transform_later(*ctrl);
+
+ Node* m = _memproj_fallthrough->clone();
+ transform_later(m);
+
+ uint alias_idx = C->get_alias_index(adr_type);
+ MergeMemNode* out_mem;
+ if (alias_idx != Compile::AliasIdxBot) {
+ out_mem = MergeMemNode::make(mem);
+ out_mem->set_memory_at(alias_idx, m);
+ } else {
+ out_mem = MergeMemNode::make(m);
+ }
+ transform_later(out_mem);
+
+ *io = _ioproj_fallthrough->clone();
+ transform_later(*io);
+
+ return out_mem;
+}
+
+// Helper function; generates code for cases requiring runtime checks.
+Node* PhaseMacroExpand::generate_checkcast_arraycopy(Node** ctrl, MergeMemNode** mem,
+ const TypePtr* adr_type,
+ Node* dest_elem_klass,
+ Node* src, Node* src_offset,
+ Node* dest, Node* dest_offset,
+ Node* copy_length, bool dest_uninitialized) {
+ if ((*ctrl)->is_top()) return NULL;
+
+ address copyfunc_addr = StubRoutines::checkcast_arraycopy(dest_uninitialized);
+ if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
+ return NULL;
+ }
+
+ // Pick out the parameters required to perform a store-check
+ // for the target array. This is an optimistic check. It will
+ // look in each non-null element's class, at the desired klass's
+ // super_check_offset, for the desired klass.
+ int sco_offset = in_bytes(Klass::super_check_offset_offset());
+ Node* p3 = basic_plus_adr(dest_elem_klass, sco_offset);
+ Node* n3 = new LoadINode(NULL, *mem /*memory(p3)*/, p3, _igvn.type(p3)->is_ptr(), TypeInt::INT, MemNode::unordered);
+ Node* check_offset = ConvI2X(transform_later(n3));
+ Node* check_value = dest_elem_klass;
+
+ Node* src_start = array_element_address(src, src_offset, T_OBJECT);
+ Node* dest_start = array_element_address(dest, dest_offset, T_OBJECT);
+
+ const TypeFunc* call_type = OptoRuntime::checkcast_arraycopy_Type();
+ Node* call = make_leaf_call(*ctrl, *mem, call_type, copyfunc_addr, "checkcast_arraycopy", adr_type,
+ src_start, dest_start, copy_length XTOP, check_offset XTOP, check_value);
+
+ finish_arraycopy_call(call, ctrl, mem, adr_type);
+
+ Node* proj = new ProjNode(call, TypeFunc::Parms);
+ transform_later(proj);
+
+ return proj;
+}
+
+// Helper function; generates code for cases requiring runtime checks.
+Node* PhaseMacroExpand::generate_generic_arraycopy(Node** ctrl, MergeMemNode** mem,
+ const TypePtr* adr_type,
+ Node* src, Node* src_offset,
+ Node* dest, Node* dest_offset,
+ Node* copy_length, bool dest_uninitialized) {
+ if ((*ctrl)->is_top()) return NULL;
+ assert(!dest_uninitialized, "Invariant");
+
+ address copyfunc_addr = StubRoutines::generic_arraycopy();
+ if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
+ return NULL;
+ }
+
+ const TypeFunc* call_type = OptoRuntime::generic_arraycopy_Type();
+ Node* call = make_leaf_call(*ctrl, *mem, call_type, copyfunc_addr, "generic_arraycopy", adr_type,
+ src, src_offset, dest, dest_offset, copy_length);
+
+ finish_arraycopy_call(call, ctrl, mem, adr_type);
+
+ Node* proj = new ProjNode(call, TypeFunc::Parms);
+ transform_later(proj);
+
+ return proj;
+}
+
+// Helper function; generates the fast out-of-line call to an arraycopy stub.
+void PhaseMacroExpand::generate_unchecked_arraycopy(Node** ctrl, MergeMemNode** mem,
+ const TypePtr* adr_type,
+ BasicType basic_elem_type,
+ bool disjoint_bases,
+ Node* src, Node* src_offset,
+ Node* dest, Node* dest_offset,
+ Node* copy_length, bool dest_uninitialized) {
+ if ((*ctrl)->is_top()) return;
+
+ Node* src_start = src;
+ Node* dest_start = dest;
+ if (src_offset != NULL || dest_offset != NULL) {
+ src_start = array_element_address(src, src_offset, basic_elem_type);
+ dest_start = array_element_address(dest, dest_offset, basic_elem_type);
+ }
+
+ // Figure out which arraycopy runtime method to call.
+ const char* copyfunc_name = "arraycopy";
+ address copyfunc_addr =
+ basictype2arraycopy(basic_elem_type, src_offset, dest_offset,
+ disjoint_bases, copyfunc_name, dest_uninitialized);
+
+ const TypeFunc* call_type = OptoRuntime::fast_arraycopy_Type();
+ Node* call = make_leaf_call(*ctrl, *mem, call_type, copyfunc_addr, copyfunc_name, adr_type,
+ src_start, dest_start, copy_length XTOP);
+
+ finish_arraycopy_call(call, ctrl, mem, adr_type);
+}
+
+void PhaseMacroExpand::expand_arraycopy_node(ArrayCopyNode *ac) {
+ Node* ctrl = ac->in(TypeFunc::Control);
+ Node* io = ac->in(TypeFunc::I_O);
+ Node* src = ac->in(ArrayCopyNode::Src);
+ Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
+ Node* dest = ac->in(ArrayCopyNode::Dest);
+ Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
+ Node* length = ac->in(ArrayCopyNode::Length);
+ MergeMemNode* merge_mem = NULL;
+
+ if (ac->is_clonebasic()) {
+ assert (src_offset == NULL && dest_offset == NULL, "for clone offsets should be null");
+ Node* mem = ac->in(TypeFunc::Memory);
+ const char* copyfunc_name = "arraycopy";
+ address copyfunc_addr =
+ basictype2arraycopy(T_LONG, NULL, NULL,
+ true, copyfunc_name, true);
+
+ const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
+ const TypeFunc* call_type = OptoRuntime::fast_arraycopy_Type();
+
+ Node* call = make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, src, dest, length XTOP);
+ transform_later(call);
+
+ _igvn.replace_node(ac, call);
+ return;
+ } else if (ac->is_copyof() || ac->is_copyofrange() || ac->is_cloneoop()) {
+ Node* mem = ac->in(TypeFunc::Memory);
+ merge_mem = MergeMemNode::make(mem);
+ transform_later(merge_mem);
+
+ RegionNode* slow_region = new RegionNode(1);
+ transform_later(slow_region);
+
+ AllocateArrayNode* alloc = NULL;
+ if (ac->is_alloc_tightly_coupled()) {
+ alloc = AllocateArrayNode::Ideal_array_allocation(dest, &_igvn);
+ assert(alloc != NULL, "expect alloc");
+ }
+
+ generate_arraycopy(ac, alloc, &ctrl, merge_mem, &io,
+ TypeAryPtr::OOPS, T_OBJECT,
+ src, src_offset, dest, dest_offset, length,
+ true, !ac->is_copyofrange());
+
+ return;
+ }
+
+ AllocateArrayNode* alloc = NULL;
+ if (ac->is_alloc_tightly_coupled()) {
+ alloc = AllocateArrayNode::Ideal_array_allocation(dest, &_igvn);
+ assert(alloc != NULL, "expect alloc");
+ }
+
+ assert(ac->is_arraycopy() || ac->is_arraycopy_notest(), "should be an arraycopy");
+
+ // Compile time checks. If any of these checks cannot be verified at compile time,
+ // we do not make a fast path for this call. Instead, we let the call remain as it
+ // is. The checks we choose to mandate at compile time are:
+ //
+ // (1) src and dest are arrays.
+ const Type* src_type = src->Value(&_igvn);
+ const Type* dest_type = dest->Value(&_igvn);
+ const TypeAryPtr* top_src = src_type->isa_aryptr();
+ const TypeAryPtr* top_dest = dest_type->isa_aryptr();
+
+ if (top_src == NULL || top_src->klass() == NULL ||
+ top_dest == NULL || top_dest->klass() == NULL) {
+ // Conservatively insert a memory barrier on all memory slices.
+ // Do not let writes into the source float below the arraycopy.
+ {
+ Node* mem = ac->in(TypeFunc::Memory);
+ insert_mem_bar(&ctrl, &mem, Op_MemBarCPUOrder);
+
+ merge_mem = MergeMemNode::make(mem);
+ transform_later(merge_mem);
+ }
+
+ // Call StubRoutines::generic_arraycopy stub.
+ Node* mem = generate_arraycopy(ac, NULL, &ctrl, merge_mem, &io,
+ TypeRawPtr::BOTTOM, T_CONFLICT,
+ src, src_offset, dest, dest_offset, length);
+
+ // Do not let reads from the destination float above the arraycopy.
+ // Since we cannot type the arrays, we don't know which slices
+ // might be affected. We could restrict this barrier only to those
+ // memory slices which pertain to array elements--but don't bother.
+ if (!InsertMemBarAfterArraycopy) {
+ // (If InsertMemBarAfterArraycopy, there is already one in place.)
+ insert_mem_bar(&ctrl, &mem, Op_MemBarCPUOrder);
+ }
+ return;
+ }
+ // (2) src and dest arrays must have elements of the same BasicType
+ // Figure out the size and type of the elements we will be copying.
+ BasicType src_elem = top_src->klass()->as_array_klass()->element_type()->basic_type();
+ BasicType dest_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type();
+ if (src_elem == T_ARRAY) src_elem = T_OBJECT;
+ if (dest_elem == T_ARRAY) dest_elem = T_OBJECT;
+
+ if (src_elem != dest_elem || dest_elem == T_VOID) {
+ // The component types are not the same or are not recognized. Punt.
+ // (But, avoid the native method wrapper to JVM_ArrayCopy.)
+ {
+ Node* mem = ac->in(TypeFunc::Memory);
+ merge_mem = generate_slow_arraycopy(ac, &ctrl, mem, &io, TypePtr::BOTTOM, src, src_offset, dest, dest_offset, length, false);
+ }
+
+ _igvn.replace_node(_memproj_fallthrough, merge_mem);
+ _igvn.replace_node(_ioproj_fallthrough, io);
+ _igvn.replace_node(_fallthroughcatchproj, ctrl);
+ return;
+ }
+
+ //---------------------------------------------------------------------------
+ // We will make a fast path for this call to arraycopy.
+
+ // We have the following tests left to perform:
+ //
+ // (3) src and dest must not be null.
+ // (4) src_offset must not be negative.
+ // (5) dest_offset must not be negative.
+ // (6) length must not be negative.
+ // (7) src_offset + length must not exceed length of src.
+ // (8) dest_offset + length must not exceed length of dest.
+ // (9) each element of an oop array must be assignable
+
+ {
+ Node* mem = ac->in(TypeFunc::Memory);
+ merge_mem = MergeMemNode::make(mem);
+ transform_later(merge_mem);
+ }
+
+ RegionNode* slow_region = new RegionNode(1);
+ transform_later(slow_region);
+
+ if (!ac->is_arraycopy_notest()) {
+ // (3) operands must not be null
+ // We currently perform our null checks with the null_check routine.
+ // This means that the null exceptions will be reported in the caller
+ // rather than (correctly) reported inside of the native arraycopy call.
+ // This should be corrected, given time. We do our null check with the
+ // stack pointer restored.
+ // null checks done library_call.cpp
+
+ // (4) src_offset must not be negative.
+ generate_negative_guard(&ctrl, src_offset, slow_region);
+
+ // (5) dest_offset must not be negative.
+ generate_negative_guard(&ctrl, dest_offset, slow_region);
+
+ // (6) length must not be negative (moved to generate_arraycopy()).
+ // generate_negative_guard(length, slow_region);
+
+ // (7) src_offset + length must not exceed length of src.
+ Node* r_adr = new AddPNode(src, src, MakeConX(arrayOopDesc::length_offset_in_bytes()));
+ transform_later(r_adr);
+ Node* alen = new LoadRangeNode(0, C->immutable_memory(), r_adr, TypeInt::POS);
+ transform_later(alen);
+ generate_limit_guard(&ctrl,
+ src_offset, length,
+ alen,
+ slow_region);
+
+ // (8) dest_offset + length must not exceed length of dest.
+ r_adr = new AddPNode(dest, dest, MakeConX(arrayOopDesc::length_offset_in_bytes()));
+ transform_later(r_adr);
+ alen = new LoadRangeNode(0, C->immutable_memory(), r_adr, TypeInt::POS);
+ transform_later(alen);
+ generate_limit_guard(&ctrl,
+ dest_offset, length,
+ alen,
+ slow_region);
+
+ // (9) each element of an oop array must be assignable
+ // The generate_arraycopy subroutine checks this.
+ }
+ // This is where the memory effects are placed:
+ const TypePtr* adr_type = TypeAryPtr::get_array_body_type(dest_elem);
+ generate_arraycopy(ac, alloc, &ctrl, merge_mem, &io,
+ adr_type, dest_elem,
+ src, src_offset, dest, dest_offset, length,
+ false, false, slow_region);
+}
--- a/hotspot/src/share/vm/opto/node.hpp Fri Aug 08 11:36:48 2014 -0700
+++ b/hotspot/src/share/vm/opto/node.hpp Mon Aug 11 14:12:51 2014 +0200
@@ -40,6 +40,7 @@
class AliasInfo;
class AllocateArrayNode;
class AllocateNode;
+class ArrayCopyNode;
class Block;
class BoolNode;
class BoxLockNode;
@@ -561,6 +562,7 @@
DEFINE_CLASS_ID(AbstractLock, Call, 3)
DEFINE_CLASS_ID(Lock, AbstractLock, 0)
DEFINE_CLASS_ID(Unlock, AbstractLock, 1)
+ DEFINE_CLASS_ID(ArrayCopy, Call, 4)
DEFINE_CLASS_ID(MultiBranch, Multi, 1)
DEFINE_CLASS_ID(PCTable, MultiBranch, 0)
DEFINE_CLASS_ID(Catch, PCTable, 0)
@@ -707,6 +709,7 @@
DEFINE_CLASS_QUERY(AddP)
DEFINE_CLASS_QUERY(Allocate)
DEFINE_CLASS_QUERY(AllocateArray)
+ DEFINE_CLASS_QUERY(ArrayCopy)
DEFINE_CLASS_QUERY(Bool)
DEFINE_CLASS_QUERY(BoxLock)
DEFINE_CLASS_QUERY(Call)
--- a/hotspot/src/share/vm/opto/phase.hpp Fri Aug 08 11:36:48 2014 -0700
+++ b/hotspot/src/share/vm/opto/phase.hpp Mon Aug 11 14:12:51 2014 +0200
@@ -27,7 +27,10 @@
#include "runtime/timer.hpp"
-class Compile;
+class IfNode;
+class MergeMemNode;
+class Node;
+class PhaseGVN;
//------------------------------Phase------------------------------------------
// Most optimizations are done in Phases. Creating a phase does any long
@@ -114,9 +117,20 @@
static elapsedTimer _t_instrSched;
static elapsedTimer _t_buildOopMaps;
#endif
+
+ // Generate a subtyping check. Takes as input the subtype and supertype.
+ // Returns 2 values: sets the default control() to the true path and
+ // returns the false path. Only reads from constant memory taken from the
+ // default memory; does not write anything. It also doesn't take in an
+ // Object; if you wish to check an Object you need to load the Object's
+ // class prior to coming here.
+ // Used in GraphKit and PhaseMacroExpand
+ static Node* gen_subtype_check(Node* subklass, Node* superklass, Node** ctrl, MergeMemNode* mem, PhaseGVN* gvn);
+
public:
Compile * C;
Phase( PhaseNumber pnum );
+
#ifndef PRODUCT
static void print_timers();
#endif
--- a/hotspot/src/share/vm/opto/phaseX.hpp Fri Aug 08 11:36:48 2014 -0700
+++ b/hotspot/src/share/vm/opto/phaseX.hpp Mon Aug 11 14:12:51 2014 +0200
@@ -390,6 +390,9 @@
// in a faster or cheaper fashion.
Node *transform( Node *n );
Node *transform_no_reclaim( Node *n );
+ virtual void record_for_igvn(Node *n) {
+ C->record_for_igvn(n);
+ }
void replace_with(PhaseGVN* gvn) {
_table.replace_with(&gvn->_table);
@@ -418,9 +421,6 @@
protected:
- // Idealize new Node 'n' with respect to its inputs and its value
- virtual Node *transform( Node *a_node );
-
// Warm up hash table, type table and initial worklist
void init_worklist( Node *a_root );
@@ -434,6 +434,10 @@
PhaseIterGVN( PhaseGVN *gvn ); // Used after Parser
PhaseIterGVN( PhaseIterGVN *igvn, const char *dummy ); // Used after +VerifyOpto
+ // Idealize new Node 'n' with respect to its inputs and its value
+ virtual Node *transform( Node *a_node );
+ virtual void record_for_igvn(Node *n) { }
+
virtual PhaseIterGVN *is_IterGVN() { return this; }
Unique_Node_List _worklist; // Iterative worklist