8137168: Replace IfNode with a new RangeCheckNode for range checks
Summary: new RangeCheckNode to enable optimization of explicit library level range checks
Reviewed-by: kvn, thartmann
--- a/hotspot/src/share/vm/opto/cfgnode.hpp Mon Nov 09 22:43:30 2015 +0000
+++ b/hotspot/src/share/vm/opto/cfgnode.hpp Mon Nov 09 11:28:31 2015 +0100
@@ -270,7 +270,6 @@
virtual uint size_of() const { return sizeof(*this); }
private:
- ProjNode* range_check_trap_proj(int& flip, Node*& l, Node*& r);
ProjNode* range_check_trap_proj() {
int flip_test = 0;
Node* l = NULL;
@@ -283,7 +282,7 @@
bool is_ctrl_folds(Node* ctrl, PhaseIterGVN* igvn);
bool has_shared_region(ProjNode* proj, ProjNode*& success, ProjNode*& fail);
bool has_only_uncommon_traps(ProjNode* proj, ProjNode*& success, ProjNode*& fail, PhaseIterGVN* igvn);
- static void merge_uncommon_traps(ProjNode* proj, ProjNode* success, ProjNode* fail, PhaseIterGVN* igvn);
+ Node* merge_uncommon_traps(ProjNode* proj, ProjNode* success, ProjNode* fail, PhaseIterGVN* igvn);
static void improve_address_types(Node* l, Node* r, ProjNode* fail, PhaseIterGVN* igvn);
bool is_cmp_with_loadrange(ProjNode* proj);
bool is_null_check(ProjNode* proj, PhaseIterGVN* igvn);
@@ -292,6 +291,12 @@
ProjNode* uncommon_trap_proj(CallStaticJavaNode*& call) const;
bool fold_compares_helper(ProjNode* proj, ProjNode* success, ProjNode* fail, PhaseIterGVN* igvn);
+protected:
+ ProjNode* range_check_trap_proj(int& flip, Node*& l, Node*& r);
+ Node* Ideal_common(PhaseGVN *phase, bool can_reshape);
+ Node* dominated_by(Node* prev_dom, PhaseIterGVN* igvn);
+ Node* search_identical(int dist);
+
public:
// Degrees of branch prediction probability by order of magnitude:
@@ -375,8 +380,6 @@
virtual const Type *Value( PhaseTransform *phase ) const;
virtual int required_outcnt() const { return 2; }
virtual const RegMask &out_RegMask() const;
- void dominated_by(Node* prev_dom, PhaseIterGVN* igvn);
- int is_range_check(Node* &range, Node* &index, jint &offset);
Node* fold_compares(PhaseIterGVN* phase);
static Node* up_one_dom(Node* curr, bool linear_only = false);
@@ -391,6 +394,20 @@
#endif
};
+class RangeCheckNode : public IfNode {
+private:
+ int is_range_check(Node* &range, Node* &index, jint &offset);
+
+public:
+ RangeCheckNode(Node* control, Node *b, float p, float fcnt)
+ : IfNode(control, b, p, fcnt) {
+ init_class_id(Class_RangeCheck);
+ }
+
+ virtual int Opcode() const;
+ virtual Node* Ideal(PhaseGVN *phase, bool can_reshape);
+};
+
class IfProjNode : public CProjNode {
public:
IfProjNode(IfNode *ifnode, uint idx) : CProjNode(ifnode,idx) {}
--- a/hotspot/src/share/vm/opto/classes.hpp Mon Nov 09 22:43:30 2015 +0000
+++ b/hotspot/src/share/vm/opto/classes.hpp Mon Nov 09 11:28:31 2015 +0100
@@ -138,6 +138,7 @@
macro(Halt)
macro(HasNegatives)
macro(If)
+macro(RangeCheck)
macro(IfFalse)
macro(IfTrue)
macro(Initialize)
--- a/hotspot/src/share/vm/opto/compile.cpp Mon Nov 09 22:43:30 2015 +0000
+++ b/hotspot/src/share/vm/opto/compile.cpp Mon Nov 09 11:28:31 2015 +0100
@@ -3181,6 +3181,13 @@
n->set_req(MemBarNode::Precedent, top());
}
break;
+ case Op_RangeCheck: {
+ RangeCheckNode* rc = n->as_RangeCheck();
+ Node* iff = new IfNode(rc->in(0), rc->in(1), rc->_prob, rc->_fcnt);
+ n->subsume_by(iff, this);
+ frc._tests.push(iff);
+ break;
+ }
default:
assert( !n->is_Call(), "" );
assert( !n->is_Mem(), "" );
@@ -3189,8 +3196,9 @@
}
// Collect CFG split points
- if (n->is_MultiBranch())
+ if (n->is_MultiBranch() && !n->is_RangeCheck()) {
frc._tests.push(n);
+ }
}
//------------------------------final_graph_reshaping_walk---------------------
--- a/hotspot/src/share/vm/opto/ifnode.cpp Mon Nov 09 22:43:30 2015 +0000
+++ b/hotspot/src/share/vm/opto/ifnode.cpp Mon Nov 09 11:28:31 2015 +0100
@@ -306,12 +306,16 @@
Node *b_c = phase->transform(new BoolNode(cmp_c,b->_test._test));
Node *b_x = phase->transform(new BoolNode(cmp_x,b->_test._test));
// Make the IfNode
- IfNode *iff_c = new IfNode(region_c,b_c,iff->_prob,iff->_fcnt);
+ IfNode* iff_c = iff->clone()->as_If();
+ iff_c->set_req(0, region_c);
+ iff_c->set_req(1, b_c);
igvn->set_type_bottom(iff_c);
igvn->_worklist.push(iff_c);
hook->init_req(2, iff_c);
- IfNode *iff_x = new IfNode(region_x,b_x,iff->_prob, iff->_fcnt);
+ IfNode* iff_x = iff->clone()->as_If();
+ iff_x->set_req(0, region_x);
+ iff_x->set_req(1, b_x);
igvn->set_type_bottom(iff_x);
igvn->_worklist.push(iff_x);
hook->init_req(3, iff_x);
@@ -496,7 +500,7 @@
// Return 0 if not a range check. Return 1 if a range check and set index and
// offset. Return 2 if we had to negate the test. Index is NULL if the check
// is versus a constant.
-int IfNode::is_range_check(Node* &range, Node* &index, jint &offset) {
+int RangeCheckNode::is_range_check(Node* &range, Node* &index, jint &offset) {
int flip_test = 0;
Node* l = NULL;
Node* r = NULL;
@@ -724,7 +728,7 @@
return ctrl != NULL &&
ctrl->is_Proj() &&
ctrl->in(0) != NULL &&
- ctrl->in(0)->is_If() &&
+ ctrl->in(0)->Opcode() == Op_If &&
ctrl->in(0)->outcnt() == 2 &&
ctrl->in(0)->as_If()->cmpi_folds(igvn) &&
// Must compare same value
@@ -972,8 +976,8 @@
if (failtype->_lo > failtype->_hi) {
// previous if determines the result of this if so
// replace Bool with constant
- igvn->hash_delete(this);
- set_req(1, igvn->intcon(success->_con));
+ igvn->_worklist.push(in(1));
+ igvn->replace_input_of(this, 1, igvn->intcon(success->_con));
return true;
}
}
@@ -992,7 +996,8 @@
Node* newbool = igvn->transform(new BoolNode(newcmp, cond));
igvn->replace_input_of(dom_iff, 1, igvn->intcon(proj->_con));
- set_req(1, newbool);
+ igvn->_worklist.push(in(1));
+ igvn->replace_input_of(this, 1, newbool);
return true;
}
@@ -1002,7 +1007,10 @@
// Merge the branches that trap for this If and the dominating If into
// a single region that branches to the uncommon trap for the
// dominating If
-void IfNode::merge_uncommon_traps(ProjNode* proj, ProjNode* success, ProjNode* fail, PhaseIterGVN* igvn) {
+Node* IfNode::merge_uncommon_traps(ProjNode* proj, ProjNode* success, ProjNode* fail, PhaseIterGVN* igvn) {
+ Node* res = this;
+ assert(success->in(0) == this, "bad projection");
+
ProjNode* otherproj = proj->other_if_proj();
CallStaticJavaNode* unc = success->is_uncommon_trap_proj(Deoptimization::Reason_none);
@@ -1038,6 +1046,8 @@
trap_request = Deoptimization::make_trap_request(Deoptimization::Reason_range_check, action);
improve_address_types(l, r, fail, igvn);
+
+ res = igvn->transform(new RangeCheckNode(in(0), in(1), _prob, _fcnt));
} else if (unc != dom_unc) {
// If we trap we won't know what CmpI would have caused the trap
// so use a special trap reason to mark this pair of CmpI nodes as
@@ -1047,6 +1057,7 @@
trap_request = Deoptimization::make_trap_request(Deoptimization::Reason_unstable_fused_if, action);
}
igvn->replace_input_of(dom_unc, TypeFunc::Parms, igvn->intcon(trap_request));
+ return res;
}
// If we are turning 2 CmpI nodes into a CmpU that follows the pattern
@@ -1240,8 +1251,7 @@
if (has_only_uncommon_traps(dom_cmp, success, fail, igvn) &&
// Next call modifies graph so must be last
fold_compares_helper(dom_cmp, success, fail, igvn)) {
- merge_uncommon_traps(dom_cmp, success, fail, igvn);
- return this;
+ return merge_uncommon_traps(dom_cmp, success, fail, igvn);
}
return NULL;
} else if (ctrl->in(0) != NULL &&
@@ -1260,8 +1270,7 @@
// Next call modifies graph so must be last
fold_compares_helper(dom_cmp, success, fail, igvn)) {
reroute_side_effect_free_unc(other_cmp, dom_cmp, igvn);
- merge_uncommon_traps(dom_cmp, success, fail, igvn);
- return this;
+ return merge_uncommon_traps(dom_cmp, success, fail, igvn);
}
}
}
@@ -1342,14 +1351,10 @@
jint off;
};
-//------------------------------Ideal------------------------------------------
-// Return a node which is more "ideal" than the current node. Strip out
-// control copies
-Node *IfNode::Ideal(PhaseGVN *phase, bool can_reshape) {
+Node* IfNode::Ideal_common(PhaseGVN *phase, bool can_reshape) {
if (remove_dead_region(phase, can_reshape)) return this;
// No Def-Use info?
if (!can_reshape) return NULL;
- PhaseIterGVN *igvn = phase->is_IterGVN();
// Don't bother trying to transform a dead if
if (in(0)->is_top()) return NULL;
@@ -1365,24 +1370,291 @@
if (idt_if != NULL) return idt_if;
// Try to split the IF
+ PhaseIterGVN *igvn = phase->is_IterGVN();
Node *s = split_if(this, igvn);
if (s != NULL) return s;
+ return NodeSentinel;
+}
+
+//------------------------------Ideal------------------------------------------
+// Return a node which is more "ideal" than the current node. Strip out
+// control copies
+Node* IfNode::Ideal(PhaseGVN *phase, bool can_reshape) {
+ Node* res = Ideal_common(phase, can_reshape);
+ if (res != NodeSentinel) {
+ return res;
+ }
+
// Check for people making a useless boolean: things like
// if( (x < y ? true : false) ) { ... }
// Replace with if( x < y ) { ... }
Node *bol2 = remove_useless_bool(this, phase);
if( bol2 ) return bol2;
+ if (in(0) == NULL) return NULL; // Dead loop?
+
+ PhaseIterGVN *igvn = phase->is_IterGVN();
+ Node* result = fold_compares(igvn);
+ if (result != NULL) {
+ return result;
+ }
+
+ // Scan for an equivalent test
+ Node *cmp;
+ int dist = 0; // Cutoff limit for search
+ int op = Opcode();
+ if( op == Op_If &&
+ (cmp=in(1)->in(1))->Opcode() == Op_CmpP ) {
+ if( cmp->in(2) != NULL && // make sure cmp is not already dead
+ cmp->in(2)->bottom_type() == TypePtr::NULL_PTR ) {
+ dist = 64; // Limit for null-pointer scans
+ } else {
+ dist = 4; // Do not bother for random pointer tests
+ }
+ } else {
+ dist = 4; // Limit for random junky scans
+ }
+
+ Node* prev_dom = search_identical(dist);
+
+ if (prev_dom == NULL) {
+ return NULL;
+ }
+
+ // Replace dominated IfNode
+ return dominated_by(prev_dom, igvn);
+}
+
+//------------------------------dominated_by-----------------------------------
+Node* IfNode::dominated_by(Node* prev_dom, PhaseIterGVN *igvn) {
+#ifndef PRODUCT
+ if (TraceIterativeGVN) {
+ tty->print(" Removing IfNode: "); this->dump();
+ }
+ if (VerifyOpto && !igvn->allow_progress()) {
+ // Found an equivalent dominating test,
+ // we can not guarantee reaching a fix-point for these during iterativeGVN
+ // since intervening nodes may not change.
+ return NULL;
+ }
+#endif
+
+ igvn->hash_delete(this); // Remove self to prevent spurious V-N
+ Node *idom = in(0);
+ // Need opcode to decide which way 'this' test goes
+ int prev_op = prev_dom->Opcode();
+ Node *top = igvn->C->top(); // Shortcut to top
+
+ // Loop predicates may have depending checks which should not
+ // be skipped. For example, range check predicate has two checks
+ // for lower and upper bounds.
+ ProjNode* unc_proj = proj_out(1 - prev_dom->as_Proj()->_con)->as_Proj();
+ if (unc_proj->is_uncommon_trap_proj(Deoptimization::Reason_predicate) != NULL)
+ prev_dom = idom;
+
+ // Now walk the current IfNode's projections.
+ // Loop ends when 'this' has no more uses.
+ for (DUIterator_Last imin, i = last_outs(imin); i >= imin; --i) {
+ Node *ifp = last_out(i); // Get IfTrue/IfFalse
+ igvn->add_users_to_worklist(ifp);
+ // Check which projection it is and set target.
+ // Data-target is either the dominating projection of the same type
+ // or TOP if the dominating projection is of opposite type.
+ // Data-target will be used as the new control edge for the non-CFG
+ // nodes like Casts and Loads.
+ Node *data_target = (ifp->Opcode() == prev_op) ? prev_dom : top;
+ // Control-target is just the If's immediate dominator or TOP.
+ Node *ctrl_target = (ifp->Opcode() == prev_op) ? idom : top;
+
+ // For each child of an IfTrue/IfFalse projection, reroute.
+ // Loop ends when projection has no more uses.
+ for (DUIterator_Last jmin, j = ifp->last_outs(jmin); j >= jmin; --j) {
+ Node* s = ifp->last_out(j); // Get child of IfTrue/IfFalse
+ if( !s->depends_only_on_test() ) {
+ // Find the control input matching this def-use edge.
+ // For Regions it may not be in slot 0.
+ uint l;
+ for( l = 0; s->in(l) != ifp; l++ ) { }
+ igvn->replace_input_of(s, l, ctrl_target);
+ } else { // Else, for control producers,
+ igvn->replace_input_of(s, 0, data_target); // Move child to data-target
+ }
+ } // End for each child of a projection
+
+ igvn->remove_dead_node(ifp);
+ } // End for each IfTrue/IfFalse child of If
+
+ // Kill the IfNode
+ igvn->remove_dead_node(this);
+
+ // Must return either the original node (now dead) or a new node
+ // (Do not return a top here, since that would break the uniqueness of top.)
+ return new ConINode(TypeInt::ZERO);
+}
+
+Node* IfNode::search_identical(int dist) {
// Setup to scan up the CFG looking for a dominating test
- Node *dom = in(0);
- Node *prev_dom = this;
+ Node* dom = in(0);
+ Node* prev_dom = this;
+ int op = Opcode();
+ // Search up the dominator tree for an If with an identical test
+ while( dom->Opcode() != op || // Not same opcode?
+ dom->in(1) != in(1) || // Not same input 1?
+ (req() == 3 && dom->in(2) != in(2)) || // Not same input 2?
+ prev_dom->in(0) != dom ) { // One path of test does not dominate?
+ if( dist < 0 ) return NULL;
+
+ dist--;
+ prev_dom = dom;
+ dom = up_one_dom( dom );
+ if( !dom ) return NULL;
+ }
+
+ // Check that we did not follow a loop back to ourselves
+ if( this == dom )
+ return NULL;
+
+ if( dist > 2 ) // Add to count of NULL checks elided
+ explicit_null_checks_elided++;
+
+ return prev_dom;
+}
+
+//------------------------------Identity---------------------------------------
+// If the test is constant & we match, then we are the input Control
+Node *IfProjNode::Identity(PhaseTransform *phase) {
+ // Can only optimize if cannot go the other way
+ const TypeTuple *t = phase->type(in(0))->is_tuple();
+ if (t == TypeTuple::IFNEITHER ||
+ // kill dead branch first otherwise the IfNode's control will
+ // have 2 control uses (the IfNode that doesn't go away because
+ // it still has uses and this branch of the
+ // If). Node::has_special_unique_user() will cause this node to
+ // be reprocessed once the dead branch is killed.
+ (always_taken(t) && in(0)->outcnt() == 1)) {
+ // IfNode control
+ return in(0)->in(0);
+ }
+ // no progress
+ return this;
+}
+
+#ifndef PRODUCT
+//-------------------------------related---------------------------------------
+// An IfProjNode's related node set consists of its input (an IfNode) including
+// the IfNode's condition, plus all of its outputs at level 1. In compact mode,
+// the restrictions for IfNode apply (see IfNode::rel).
+void IfProjNode::related(GrowableArray<Node*> *in_rel, GrowableArray<Node*> *out_rel, bool compact) const {
+ Node* ifNode = this->in(0);
+ in_rel->append(ifNode);
+ if (compact) {
+ ifNode->collect_nodes(in_rel, 3, false, true);
+ } else {
+ ifNode->collect_nodes_in_all_data(in_rel, false);
+ }
+ this->collect_nodes(out_rel, -1, false, false);
+}
+
+//------------------------------dump_spec--------------------------------------
+void IfNode::dump_spec(outputStream *st) const {
+ st->print("P=%f, C=%f",_prob,_fcnt);
+}
+
+//-------------------------------related---------------------------------------
+// For an IfNode, the set of related output nodes is just the output nodes till
+// depth 2, i.e, the IfTrue/IfFalse projection nodes plus the nodes they refer.
+// The related input nodes contain no control nodes, but all data nodes
+// pertaining to the condition. In compact mode, the input nodes are collected
+// up to a depth of 3.
+void IfNode::related(GrowableArray <Node *> *in_rel, GrowableArray <Node *> *out_rel, bool compact) const {
+ if (compact) {
+ this->collect_nodes(in_rel, 3, false, true);
+ } else {
+ this->collect_nodes_in_all_data(in_rel, false);
+ }
+ this->collect_nodes(out_rel, -2, false, false);
+}
+#endif
+
+//------------------------------idealize_test----------------------------------
+// Try to canonicalize tests better. Peek at the Cmp/Bool/If sequence and
+// come up with a canonical sequence. Bools getting 'eq', 'gt' and 'ge' forms
+// converted to 'ne', 'le' and 'lt' forms. IfTrue/IfFalse get swapped as
+// needed.
+static IfNode* idealize_test(PhaseGVN* phase, IfNode* iff) {
+ assert(iff->in(0) != NULL, "If must be live");
+
+ if (iff->outcnt() != 2) return NULL; // Malformed projections.
+ Node* old_if_f = iff->proj_out(false);
+ Node* old_if_t = iff->proj_out(true);
+
+ // CountedLoopEnds want the back-control test to be TRUE, irregardless of
+ // whether they are testing a 'gt' or 'lt' condition. The 'gt' condition
+ // happens in count-down loops
+ if (iff->is_CountedLoopEnd()) return NULL;
+ if (!iff->in(1)->is_Bool()) return NULL; // Happens for partially optimized IF tests
+ BoolNode *b = iff->in(1)->as_Bool();
+ BoolTest bt = b->_test;
+ // Test already in good order?
+ if( bt.is_canonical() )
+ return NULL;
+
+ // Flip test to be canonical. Requires flipping the IfFalse/IfTrue and
+ // cloning the IfNode.
+ Node* new_b = phase->transform( new BoolNode(b->in(1), bt.negate()) );
+ if( !new_b->is_Bool() ) return NULL;
+ b = new_b->as_Bool();
+
+ PhaseIterGVN *igvn = phase->is_IterGVN();
+ assert( igvn, "Test is not canonical in parser?" );
+
+ // The IF node never really changes, but it needs to be cloned
+ iff = iff->clone()->as_If();
+ iff->set_req(1, b);
+ iff->_prob = 1.0-iff->_prob;
+
+ Node *prior = igvn->hash_find_insert(iff);
+ if( prior ) {
+ igvn->remove_dead_node(iff);
+ iff = (IfNode*)prior;
+ } else {
+ // Cannot call transform on it just yet
+ igvn->set_type_bottom(iff);
+ }
+ igvn->_worklist.push(iff);
+
+ // Now handle projections. Cloning not required.
+ Node* new_if_f = (Node*)(new IfFalseNode( iff ));
+ Node* new_if_t = (Node*)(new IfTrueNode ( iff ));
+
+ igvn->register_new_node_with_optimizer(new_if_f);
+ igvn->register_new_node_with_optimizer(new_if_t);
+ // Flip test, so flip trailing control
+ igvn->replace_node(old_if_f, new_if_t);
+ igvn->replace_node(old_if_t, new_if_f);
+
+ // Progress
+ return iff;
+}
+
+Node* RangeCheckNode::Ideal(PhaseGVN *phase, bool can_reshape) {
+ Node* res = Ideal_common(phase, can_reshape);
+ if (res != NodeSentinel) {
+ return res;
+ }
+
+ PhaseIterGVN *igvn = phase->is_IterGVN();
+ // Setup to scan up the CFG looking for a dominating test
+ Node* prev_dom = this;
// Check for range-check vs other kinds of tests
- Node *index1, *range1;
+ Node* index1;
+ Node* range1;
jint offset1;
int flip1 = is_range_check(range1, index1, offset1);
- if( flip1 ) {
+ if (flip1) {
+ Node* dom = in(0);
// Try to remove extra range checks. All 'up_one_dom' gives up at merges
// so all checks we inspect post-dominate the top-most check we find.
// If we are going to fail the current check and we reach the top check
@@ -1403,13 +1675,14 @@
// Scan for the top checks and collect range of offsets
for (int dist = 0; dist < 999; dist++) { // Range-Check scan limit
- if (dom->Opcode() == Op_If && // Not same opcode?
+ if (dom->Opcode() == Op_RangeCheck && // Not same opcode?
prev_dom->in(0) == dom) { // One path of test does dominate?
if (dom == this) return NULL; // dead loop
// See if this is a range check
- Node *index2, *range2;
+ Node* index2;
+ Node* range2;
jint offset2;
- int flip2 = dom->as_If()->is_range_check(range2, index2, offset2);
+ int flip2 = dom->as_RangeCheck()->is_range_check(range2, index2, offset2);
// See if this is a _matching_ range check, checking against
// the same array bounds.
if (flip2 == flip1 && range2 == range1 && index2 == index1 &&
@@ -1517,237 +1790,14 @@
prev_dom = rc0.ctl;
}
}
-
- } else { // Scan for an equivalent test
-
- Node *cmp;
- int dist = 0; // Cutoff limit for search
- int op = Opcode();
- if( op == Op_If &&
- (cmp=in(1)->in(1))->Opcode() == Op_CmpP ) {
- if( cmp->in(2) != NULL && // make sure cmp is not already dead
- cmp->in(2)->bottom_type() == TypePtr::NULL_PTR ) {
- dist = 64; // Limit for null-pointer scans
- } else {
- dist = 4; // Do not bother for random pointer tests
- }
- } else {
- dist = 4; // Limit for random junky scans
- }
-
- // Normal equivalent-test check.
- if( !dom ) return NULL; // Dead loop?
-
- Node* result = fold_compares(igvn);
- if (result != NULL) {
- return result;
- }
+ } else {
+ prev_dom = search_identical(4);
- // Search up the dominator tree for an If with an identical test
- while( dom->Opcode() != op || // Not same opcode?
- dom->in(1) != in(1) || // Not same input 1?
- (req() == 3 && dom->in(2) != in(2)) || // Not same input 2?
- prev_dom->in(0) != dom ) { // One path of test does not dominate?
- if( dist < 0 ) return NULL;
-
- dist--;
- prev_dom = dom;
- dom = up_one_dom( dom );
- if( !dom ) return NULL;
- }
-
- // Check that we did not follow a loop back to ourselves
- if( this == dom )
+ if (prev_dom == NULL) {
return NULL;
-
- if( dist > 2 ) // Add to count of NULL checks elided
- explicit_null_checks_elided++;
-
- } // End of Else scan for an equivalent test
-
- // Hit! Remove this IF
-#ifndef PRODUCT
- if( TraceIterativeGVN ) {
- tty->print(" Removing IfNode: "); this->dump();
+ }
}
- if( VerifyOpto && !phase->allow_progress() ) {
- // Found an equivalent dominating test,
- // we can not guarantee reaching a fix-point for these during iterativeGVN
- // since intervening nodes may not change.
- return NULL;
- }
-#endif
// Replace dominated IfNode
- dominated_by( prev_dom, igvn );
-
- // Must return either the original node (now dead) or a new node
- // (Do not return a top here, since that would break the uniqueness of top.)
- return new ConINode(TypeInt::ZERO);
-}
-
-//------------------------------dominated_by-----------------------------------
-void IfNode::dominated_by( Node *prev_dom, PhaseIterGVN *igvn ) {
- igvn->hash_delete(this); // Remove self to prevent spurious V-N
- Node *idom = in(0);
- // Need opcode to decide which way 'this' test goes
- int prev_op = prev_dom->Opcode();
- Node *top = igvn->C->top(); // Shortcut to top
-
- // Loop predicates may have depending checks which should not
- // be skipped. For example, range check predicate has two checks
- // for lower and upper bounds.
- ProjNode* unc_proj = proj_out(1 - prev_dom->as_Proj()->_con)->as_Proj();
- if (unc_proj->is_uncommon_trap_proj(Deoptimization::Reason_predicate) != NULL)
- prev_dom = idom;
-
- // Now walk the current IfNode's projections.
- // Loop ends when 'this' has no more uses.
- for (DUIterator_Last imin, i = last_outs(imin); i >= imin; --i) {
- Node *ifp = last_out(i); // Get IfTrue/IfFalse
- igvn->add_users_to_worklist(ifp);
- // Check which projection it is and set target.
- // Data-target is either the dominating projection of the same type
- // or TOP if the dominating projection is of opposite type.
- // Data-target will be used as the new control edge for the non-CFG
- // nodes like Casts and Loads.
- Node *data_target = (ifp->Opcode() == prev_op) ? prev_dom : top;
- // Control-target is just the If's immediate dominator or TOP.
- Node *ctrl_target = (ifp->Opcode() == prev_op) ? idom : top;
-
- // For each child of an IfTrue/IfFalse projection, reroute.
- // Loop ends when projection has no more uses.
- for (DUIterator_Last jmin, j = ifp->last_outs(jmin); j >= jmin; --j) {
- Node* s = ifp->last_out(j); // Get child of IfTrue/IfFalse
- if( !s->depends_only_on_test() ) {
- // Find the control input matching this def-use edge.
- // For Regions it may not be in slot 0.
- uint l;
- for( l = 0; s->in(l) != ifp; l++ ) { }
- igvn->replace_input_of(s, l, ctrl_target);
- } else { // Else, for control producers,
- igvn->replace_input_of(s, 0, data_target); // Move child to data-target
- }
- } // End for each child of a projection
-
- igvn->remove_dead_node(ifp);
- } // End for each IfTrue/IfFalse child of If
-
- // Kill the IfNode
- igvn->remove_dead_node(this);
-}
-
-//------------------------------Identity---------------------------------------
-// If the test is constant & we match, then we are the input Control
-Node *IfProjNode::Identity(PhaseTransform *phase) {
- // Can only optimize if cannot go the other way
- const TypeTuple *t = phase->type(in(0))->is_tuple();
- if (t == TypeTuple::IFNEITHER ||
- // kill dead branch first otherwise the IfNode's control will
- // have 2 control uses (the IfNode that doesn't go away because
- // it still has uses and this branch of the
- // If). Node::has_special_unique_user() will cause this node to
- // be reprocessed once the dead branch is killed.
- (always_taken(t) && in(0)->outcnt() == 1)) {
- // IfNode control
- return in(0)->in(0);
- }
- // no progress
- return this;
+ return dominated_by(prev_dom, igvn);
}
-
-#ifndef PRODUCT
-//-------------------------------related---------------------------------------
-// An IfProjNode's related node set consists of its input (an IfNode) including
-// the IfNode's condition, plus all of its outputs at level 1. In compact mode,
-// the restrictions for IfNode apply (see IfNode::rel).
-void IfProjNode::related(GrowableArray<Node*> *in_rel, GrowableArray<Node*> *out_rel, bool compact) const {
- Node* ifNode = this->in(0);
- in_rel->append(ifNode);
- if (compact) {
- ifNode->collect_nodes(in_rel, 3, false, true);
- } else {
- ifNode->collect_nodes_in_all_data(in_rel, false);
- }
- this->collect_nodes(out_rel, -1, false, false);
-}
-
-//------------------------------dump_spec--------------------------------------
-void IfNode::dump_spec(outputStream *st) const {
- st->print("P=%f, C=%f",_prob,_fcnt);
-}
-
-//-------------------------------related---------------------------------------
-// For an IfNode, the set of related output nodes is just the output nodes till
-// depth 2, i.e, the IfTrue/IfFalse projection nodes plus the nodes they refer.
-// The related input nodes contain no control nodes, but all data nodes
-// pertaining to the condition. In compact mode, the input nodes are collected
-// up to a depth of 3.
-void IfNode::related(GrowableArray <Node *> *in_rel, GrowableArray <Node *> *out_rel, bool compact) const {
- if (compact) {
- this->collect_nodes(in_rel, 3, false, true);
- } else {
- this->collect_nodes_in_all_data(in_rel, false);
- }
- this->collect_nodes(out_rel, -2, false, false);
-}
-#endif
-
-//------------------------------idealize_test----------------------------------
-// Try to canonicalize tests better. Peek at the Cmp/Bool/If sequence and
-// come up with a canonical sequence. Bools getting 'eq', 'gt' and 'ge' forms
-// converted to 'ne', 'le' and 'lt' forms. IfTrue/IfFalse get swapped as
-// needed.
-static IfNode* idealize_test(PhaseGVN* phase, IfNode* iff) {
- assert(iff->in(0) != NULL, "If must be live");
-
- if (iff->outcnt() != 2) return NULL; // Malformed projections.
- Node* old_if_f = iff->proj_out(false);
- Node* old_if_t = iff->proj_out(true);
-
- // CountedLoopEnds want the back-control test to be TRUE, irregardless of
- // whether they are testing a 'gt' or 'lt' condition. The 'gt' condition
- // happens in count-down loops
- if (iff->is_CountedLoopEnd()) return NULL;
- if (!iff->in(1)->is_Bool()) return NULL; // Happens for partially optimized IF tests
- BoolNode *b = iff->in(1)->as_Bool();
- BoolTest bt = b->_test;
- // Test already in good order?
- if( bt.is_canonical() )
- return NULL;
-
- // Flip test to be canonical. Requires flipping the IfFalse/IfTrue and
- // cloning the IfNode.
- Node* new_b = phase->transform( new BoolNode(b->in(1), bt.negate()) );
- if( !new_b->is_Bool() ) return NULL;
- b = new_b->as_Bool();
-
- PhaseIterGVN *igvn = phase->is_IterGVN();
- assert( igvn, "Test is not canonical in parser?" );
-
- // The IF node never really changes, but it needs to be cloned
- iff = new IfNode( iff->in(0), b, 1.0-iff->_prob, iff->_fcnt);
-
- Node *prior = igvn->hash_find_insert(iff);
- if( prior ) {
- igvn->remove_dead_node(iff);
- iff = (IfNode*)prior;
- } else {
- // Cannot call transform on it just yet
- igvn->set_type_bottom(iff);
- }
- igvn->_worklist.push(iff);
-
- // Now handle projections. Cloning not required.
- Node* new_if_f = (Node*)(new IfFalseNode( iff ));
- Node* new_if_t = (Node*)(new IfTrueNode ( iff ));
-
- igvn->register_new_node_with_optimizer(new_if_f);
- igvn->register_new_node_with_optimizer(new_if_t);
- // Flip test, so flip trailing control
- igvn->replace_node(old_if_f, new_if_t);
- igvn->replace_node(old_if_t, new_if_f);
-
- // Progress
- return iff;
-}
--- a/hotspot/src/share/vm/opto/loopPredicate.cpp Mon Nov 09 22:43:30 2015 +0000
+++ b/hotspot/src/share/vm/opto/loopPredicate.cpp Mon Nov 09 11:28:31 2015 +0100
@@ -91,7 +91,8 @@
// The true projecttion (if_cont) of the new_iff is returned.
// This code is also used to clone predicates to cloned loops.
ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry,
- Deoptimization::DeoptReason reason) {
+ Deoptimization::DeoptReason reason,
+ int opcode) {
assert(cont_proj->is_uncommon_trap_if_pattern(reason), "must be a uct if pattern!");
IfNode* iff = cont_proj->in(0)->as_If();
@@ -133,8 +134,13 @@
}
// Create new_iff
IdealLoopTree* lp = get_loop(entry);
- IfNode *new_iff = iff->clone()->as_If();
- new_iff->set_req(0, entry);
+ IfNode* new_iff = NULL;
+ if (opcode == Op_If) {
+ new_iff = new IfNode(entry, iff->in(1), iff->_prob, iff->_fcnt);
+ } else {
+ assert(opcode == Op_RangeCheck, "no other if variant here");
+ new_iff = new RangeCheckNode(entry, iff->in(1), iff->_prob, iff->_fcnt);
+ }
register_control(new_iff, lp, entry);
Node *if_cont = new IfTrueNode(new_iff);
Node *if_uct = new IfFalseNode(new_iff);
@@ -183,7 +189,8 @@
//------------------------------create_new_if_for_predicate------------------------
// Create a new if below new_entry for the predicate to be cloned (IGVN optimization)
ProjNode* PhaseIterGVN::create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry,
- Deoptimization::DeoptReason reason) {
+ Deoptimization::DeoptReason reason,
+ int opcode) {
assert(new_entry != 0, "only used for clone predicate");
assert(cont_proj->is_uncommon_trap_if_pattern(reason), "must be a uct if pattern!");
IfNode* iff = cont_proj->in(0)->as_If();
@@ -208,8 +215,13 @@
}
// Create new_iff in new location.
- IfNode *new_iff = iff->clone()->as_If();
- new_iff->set_req(0, new_entry);
+ IfNode* new_iff = NULL;
+ if (opcode == Op_If) {
+ new_iff = new IfNode(new_entry, iff->in(1), iff->_prob, iff->_fcnt);
+ } else {
+ assert(opcode == Op_RangeCheck, "no other if variant here");
+ new_iff = new RangeCheckNode(new_entry, iff->in(1), iff->_prob, iff->_fcnt);
+ }
register_new_node_with_optimizer(new_iff);
Node *if_cont = new IfTrueNode(new_iff);
@@ -249,9 +261,9 @@
PhaseIterGVN* igvn) {
ProjNode* new_predicate_proj;
if (loop_phase != NULL) {
- new_predicate_proj = loop_phase->create_new_if_for_predicate(predicate_proj, new_entry, reason);
+ new_predicate_proj = loop_phase->create_new_if_for_predicate(predicate_proj, new_entry, reason, Op_If);
} else {
- new_predicate_proj = igvn->create_new_if_for_predicate(predicate_proj, new_entry, reason);
+ new_predicate_proj = igvn->create_new_if_for_predicate(predicate_proj, new_entry, reason, Op_If);
}
IfNode* iff = new_predicate_proj->in(0)->as_If();
Node* ctrl = iff->in(0);
@@ -714,7 +726,8 @@
while (current_proj != head) {
if (loop == get_loop(current_proj) && // still in the loop ?
current_proj->is_Proj() && // is a projection ?
- current_proj->in(0)->Opcode() == Op_If) { // is a if projection ?
+ (current_proj->in(0)->Opcode() == Op_If ||
+ current_proj->in(0)->Opcode() == Op_RangeCheck)) { // is a if projection ?
if_proj_list.push(current_proj);
}
current_proj = idom(current_proj);
@@ -753,7 +766,8 @@
if (invar.is_invariant(bol)) {
// Invariant test
new_predicate_proj = create_new_if_for_predicate(predicate_proj, NULL,
- Deoptimization::Reason_predicate);
+ Deoptimization::Reason_predicate,
+ iff->Opcode());
Node* ctrl = new_predicate_proj->in(0)->as_If()->in(0);
BoolNode* new_predicate_bol = invar.clone(bol, ctrl)->as_Bool();
@@ -797,8 +811,8 @@
// lower_bound test will dominate the upper bound test and all
// cloned or created nodes will use the lower bound test as
// their declared control.
- ProjNode* lower_bound_proj = create_new_if_for_predicate(predicate_proj, NULL, Deoptimization::Reason_predicate);
- ProjNode* upper_bound_proj = create_new_if_for_predicate(predicate_proj, NULL, Deoptimization::Reason_predicate);
+ ProjNode* lower_bound_proj = create_new_if_for_predicate(predicate_proj, NULL, Deoptimization::Reason_predicate, iff->Opcode());
+ ProjNode* upper_bound_proj = create_new_if_for_predicate(predicate_proj, NULL, Deoptimization::Reason_predicate, iff->Opcode());
assert(upper_bound_proj->in(0)->as_If()->in(0) == lower_bound_proj, "should dominate");
Node *ctrl = lower_bound_proj->in(0)->as_If()->in(0);
--- a/hotspot/src/share/vm/opto/loopTransform.cpp Mon Nov 09 22:43:30 2015 +0000
+++ b/hotspot/src/share/vm/opto/loopTransform.cpp Mon Nov 09 11:28:31 2015 +0100
@@ -290,7 +290,7 @@
if (ctrl->is_top())
return false; // Found dead test on live IF? No peeling!
// Standard IF only has one input value to check for loop invariance
- assert( test->Opcode() == Op_If || test->Opcode() == Op_CountedLoopEnd, "Check this code when new subtype is added");
+ assert(test->Opcode() == Op_If || test->Opcode() == Op_CountedLoopEnd || test->Opcode() == Op_RangeCheck, "Check this code when new subtype is added");
// Condition is not a member of this loop?
if( !is_member(phase->get_loop(ctrl)) &&
is_loop_exit(test) )
@@ -856,7 +856,8 @@
// loop-invariant.
for (uint i = 0; i < _body.size(); i++) {
Node *iff = _body[i];
- if (iff->Opcode() == Op_If) { // Test?
+ if (iff->Opcode() == Op_If ||
+ iff->Opcode() == Op_RangeCheck) { // Test?
// Comparing trip+off vs limit
Node *bol = iff->in(1);
@@ -2035,8 +2036,8 @@
// loop-invariant.
for( uint i = 0; i < loop->_body.size(); i++ ) {
Node *iff = loop->_body[i];
- if( iff->Opcode() == Op_If ) { // Test?
-
+ if (iff->Opcode() == Op_If ||
+ iff->Opcode() == Op_RangeCheck) { // Test?
// Test is an IfNode, has 2 projections. If BOTH are in the loop
// we need loop unswitching instead of iteration splitting.
Node *exit = loop->is_loop_exit(iff);
--- a/hotspot/src/share/vm/opto/loopUnswitch.cpp Mon Nov 09 22:43:30 2015 +0000
+++ b/hotspot/src/share/vm/opto/loopUnswitch.cpp Mon Nov 09 11:28:31 2015 +0100
@@ -132,7 +132,7 @@
head->as_CountedLoop()->set_normal_loop();
}
- ProjNode* proj_true = create_slow_version_of_loop(loop, old_new);
+ ProjNode* proj_true = create_slow_version_of_loop(loop, old_new, unswitch_iff->Opcode());
#ifdef ASSERT
Node* uniqc = proj_true->unique_ctrl_out();
@@ -222,7 +222,8 @@
// and inserting an if to select fast-slow versions.
// Return control projection of the entry to the fast version.
ProjNode* PhaseIdealLoop::create_slow_version_of_loop(IdealLoopTree *loop,
- Node_List &old_new) {
+ Node_List &old_new,
+ int opcode) {
LoopNode* head = loop->_head->as_Loop();
bool counted_loop = head->is_CountedLoop();
Node* entry = head->in(LoopNode::EntryControl);
@@ -235,7 +236,8 @@
register_node(opq, outer_loop, entry, dom_depth(entry));
Node *bol = new Conv2BNode(opq);
register_node(bol, outer_loop, entry, dom_depth(entry));
- IfNode* iff = new IfNode(entry, bol, PROB_MAX, COUNT_UNKNOWN);
+ IfNode* iff = (opcode == Op_RangeCheck) ? new RangeCheckNode(entry, bol, PROB_MAX, COUNT_UNKNOWN) :
+ new IfNode(entry, bol, PROB_MAX, COUNT_UNKNOWN);
register_node(iff, outer_loop, entry, dom_depth(entry));
ProjNode* iffast = new IfTrueNode(iff);
register_node(iffast, outer_loop, iff, dom_depth(iff));
--- a/hotspot/src/share/vm/opto/loopnode.hpp Mon Nov 09 22:43:30 2015 +0000
+++ b/hotspot/src/share/vm/opto/loopnode.hpp Mon Nov 09 11:28:31 2015 +0100
@@ -916,7 +916,8 @@
// Create a new if above the uncommon_trap_if_pattern for the predicate to be promoted
ProjNode* create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry,
- Deoptimization::DeoptReason reason);
+ Deoptimization::DeoptReason reason,
+ int opcode);
void register_control(Node* n, IdealLoopTree *loop, Node* pred);
// Clone loop predicates to cloned loops (peeled, unswitched)
@@ -966,7 +967,8 @@
// Create a slow version of the loop by cloning the loop
// and inserting an if to select fast-slow versions.
ProjNode* create_slow_version_of_loop(IdealLoopTree *loop,
- Node_List &old_new);
+ Node_List &old_new,
+ int opcode);
// Clone a loop and return the clone head (clone_loop_head).
// Added nodes include int(1), int(0) - disconnected, If, IfTrue, IfFalse,
--- a/hotspot/src/share/vm/opto/loopopts.cpp Mon Nov 09 22:43:30 2015 +0000
+++ b/hotspot/src/share/vm/opto/loopopts.cpp Mon Nov 09 11:28:31 2015 +0100
@@ -206,7 +206,7 @@
// prevdom is the dominating projection of the dominating test.
assert( iff->is_If(), "" );
- assert( iff->Opcode() == Op_If || iff->Opcode() == Op_CountedLoopEnd, "Check this code when new subtype is added");
+ assert(iff->Opcode() == Op_If || iff->Opcode() == Op_CountedLoopEnd || iff->Opcode() == Op_RangeCheck, "Check this code when new subtype is added");
int pop = prevdom->Opcode();
assert( pop == Op_IfFalse || pop == Op_IfTrue, "" );
if (flip) {
@@ -1123,7 +1123,8 @@
int n_op = n->Opcode();
// Check for an IF being dominated by another IF same test
- if (n_op == Op_If) {
+ if (n_op == Op_If ||
+ n_op == Op_RangeCheck) {
Node *bol = n->in(1);
uint max = bol->outcnt();
// Check for same test used more than once?
@@ -1945,7 +1946,10 @@
BoolNode* bol = new BoolNode(cmp, relop);
register_node(bol, loop, proj2, ddepth);
- IfNode* new_if = new IfNode(proj2, bol, iff->_prob, iff->_fcnt);
+ int opcode = iff->Opcode();
+ assert(opcode == Op_If || opcode == Op_RangeCheck, "unexpected opcode");
+ IfNode* new_if = (opcode == Op_If) ? new IfNode(proj2, bol, iff->_prob, iff->_fcnt):
+ new RangeCheckNode(proj2, bol, iff->_prob, iff->_fcnt);
register_node(new_if, loop, proj2, ddepth);
proj->set_req(0, new_if); // reattach
--- a/hotspot/src/share/vm/opto/multnode.cpp Mon Nov 09 22:43:30 2015 +0000
+++ b/hotspot/src/share/vm/opto/multnode.cpp Mon Nov 09 11:28:31 2015 +0100
@@ -44,14 +44,14 @@
//------------------------------proj_out---------------------------------------
// Get a named projection
ProjNode* MultiNode::proj_out(uint which_proj) const {
- assert(Opcode() != Op_If || which_proj == (uint)true || which_proj == (uint)false, "must be 1 or 0");
- assert(Opcode() != Op_If || outcnt() == 2, "bad if #1");
+ assert((Opcode() != Op_If && Opcode() != Op_RangeCheck) || which_proj == (uint)true || which_proj == (uint)false, "must be 1 or 0");
+ assert((Opcode() != Op_If && Opcode() != Op_RangeCheck) || outcnt() == 2, "bad if #1");
for( DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++ ) {
Node *p = fast_out(i);
if (p->is_Proj()) {
ProjNode *proj = p->as_Proj();
if (proj->_con == which_proj) {
- assert(Opcode() != Op_If || proj->Opcode() == (which_proj?Op_IfTrue:Op_IfFalse), "bad if #2");
+ assert((Opcode() != Op_If && Opcode() != Op_RangeCheck) || proj->Opcode() == (which_proj ? Op_IfTrue : Op_IfFalse), "bad if #2");
return proj;
}
} else {
--- a/hotspot/src/share/vm/opto/node.hpp Mon Nov 09 22:43:30 2015 +0000
+++ b/hotspot/src/share/vm/opto/node.hpp Mon Nov 09 11:28:31 2015 +0100
@@ -125,6 +125,7 @@
class PhiNode;
class Pipeline;
class ProjNode;
+class RangeCheckNode;
class RegMask;
class RegionNode;
class RootNode;
@@ -584,6 +585,7 @@
DEFINE_CLASS_ID(Jump, PCTable, 1)
DEFINE_CLASS_ID(If, MultiBranch, 1)
DEFINE_CLASS_ID(CountedLoopEnd, If, 0)
+ DEFINE_CLASS_ID(RangeCheck, If, 1)
DEFINE_CLASS_ID(NeverBranch, MultiBranch, 2)
DEFINE_CLASS_ID(Start, Multi, 2)
DEFINE_CLASS_ID(MemBar, Multi, 3)
@@ -758,6 +760,7 @@
DEFINE_CLASS_QUERY(FastLock)
DEFINE_CLASS_QUERY(FastUnlock)
DEFINE_CLASS_QUERY(If)
+ DEFINE_CLASS_QUERY(RangeCheck)
DEFINE_CLASS_QUERY(IfFalse)
DEFINE_CLASS_QUERY(IfTrue)
DEFINE_CLASS_QUERY(Initialize)
--- a/hotspot/src/share/vm/opto/parse2.cpp Mon Nov 09 22:43:30 2015 +0000
+++ b/hotspot/src/share/vm/opto/parse2.cpp Mon Nov 09 11:28:31 2015 +0100
@@ -136,8 +136,16 @@
BoolTest::mask btest = BoolTest::lt;
tst = _gvn.transform( new BoolNode(chk, btest) );
}
+ RangeCheckNode* rc = new RangeCheckNode(control(), tst, PROB_MAX, COUNT_UNKNOWN);
+ _gvn.set_type(rc, rc->Value(&_gvn));
+ if (!tst->is_Con()) {
+ record_for_igvn(rc);
+ }
+ set_control(_gvn.transform(new IfTrueNode(rc)));
// Branch to failure if out of bounds
- { BuildCutout unless(this, tst, PROB_MAX);
+ {
+ PreserveJVMState pjvms(this);
+ set_control(_gvn.transform(new IfFalseNode(rc)));
if (C->allow_range_check_smearing()) {
// Do not use builtin_throw, since range checks are sometimes
// made more stringent by an optimistic transformation.
--- a/hotspot/src/share/vm/opto/phaseX.hpp Mon Nov 09 22:43:30 2015 +0000
+++ b/hotspot/src/share/vm/opto/phaseX.hpp Mon Nov 09 11:28:31 2015 +0100
@@ -521,7 +521,8 @@
Node* clone_loop_predicates(Node* old_entry, Node* new_entry, bool clone_limit_check);
// Create a new if below new_entry for the predicate to be cloned
ProjNode* create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry,
- Deoptimization::DeoptReason reason);
+ Deoptimization::DeoptReason reason,
+ int opcode);
void remove_speculative_types();
void check_no_speculative_types() {