130 // Need to revert back to normal loop |
130 // Need to revert back to normal loop |
131 if (head->is_CountedLoop() && !head->as_CountedLoop()->is_normal_loop()) { |
131 if (head->is_CountedLoop() && !head->as_CountedLoop()->is_normal_loop()) { |
132 head->as_CountedLoop()->set_normal_loop(); |
132 head->as_CountedLoop()->set_normal_loop(); |
133 } |
133 } |
134 |
134 |
135 ProjNode* proj_true = create_slow_version_of_loop(loop, old_new, unswitch_iff->Opcode()); |
135 ProjNode* proj_true = create_slow_version_of_loop(loop, old_new, unswitch_iff->Opcode(), CloneIncludesStripMined); |
136 |
136 |
137 #ifdef ASSERT |
137 #ifdef ASSERT |
138 Node* uniqc = proj_true->unique_ctrl_out(); |
138 Node* uniqc = proj_true->unique_ctrl_out(); |
139 Node* entry = head->in(LoopNode::EntryControl); |
139 Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl); |
140 Node* predicate = find_predicate(entry); |
140 Node* predicate = find_predicate(entry); |
141 if (predicate != NULL && UseLoopPredicate) { |
141 if (predicate != NULL && UseLoopPredicate) { |
142 // We may have two predicates, find first. |
142 // We may have two predicates, find first. |
143 entry = find_predicate(entry->in(0)->in(0)); |
143 entry = find_predicate(entry->in(0)->in(0)); |
144 if (entry != NULL) predicate = entry; |
144 if (entry != NULL) predicate = entry; |
145 } |
145 } |
146 if (predicate != NULL) predicate = predicate->in(0); |
146 if (predicate != NULL) predicate = predicate->in(0); |
147 assert(proj_true->is_IfTrue() && |
147 assert(proj_true->is_IfTrue() && |
148 (predicate == NULL && uniqc == head || |
148 (predicate == NULL && uniqc == head && !head->is_strip_mined() || |
|
149 predicate == NULL && uniqc == head->in(LoopNode::EntryControl) && head->is_strip_mined() || |
149 predicate != NULL && uniqc == predicate), "by construction"); |
150 predicate != NULL && uniqc == predicate), "by construction"); |
150 #endif |
151 #endif |
151 // Increment unswitch count |
152 // Increment unswitch count |
152 LoopNode* head_clone = old_new[head->_idx]->as_Loop(); |
153 LoopNode* head_clone = old_new[head->_idx]->as_Loop(); |
153 int nct = head->unswitch_count() + 1; |
154 int nct = head->unswitch_count() + 1; |
221 // Create a slow version of the loop by cloning the loop |
222 // Create a slow version of the loop by cloning the loop |
222 // and inserting an if to select fast-slow versions. |
223 // and inserting an if to select fast-slow versions. |
223 // Return control projection of the entry to the fast version. |
224 // Return control projection of the entry to the fast version. |
224 ProjNode* PhaseIdealLoop::create_slow_version_of_loop(IdealLoopTree *loop, |
225 ProjNode* PhaseIdealLoop::create_slow_version_of_loop(IdealLoopTree *loop, |
225 Node_List &old_new, |
226 Node_List &old_new, |
226 int opcode) { |
227 int opcode, |
|
228 CloneLoopMode mode) { |
227 LoopNode* head = loop->_head->as_Loop(); |
229 LoopNode* head = loop->_head->as_Loop(); |
228 bool counted_loop = head->is_CountedLoop(); |
230 bool counted_loop = head->is_CountedLoop(); |
229 Node* entry = head->in(LoopNode::EntryControl); |
231 Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl); |
230 _igvn.rehash_node_delayed(entry); |
232 _igvn.rehash_node_delayed(entry); |
231 IdealLoopTree* outer_loop = loop->_parent; |
233 IdealLoopTree* outer_loop = loop->_parent; |
|
234 |
|
235 head->verify_strip_mined(1); |
232 |
236 |
233 Node *cont = _igvn.intcon(1); |
237 Node *cont = _igvn.intcon(1); |
234 set_ctrl(cont, C->root()); |
238 set_ctrl(cont, C->root()); |
235 Node* opq = new Opaque1Node(C, cont); |
239 Node* opq = new Opaque1Node(C, cont); |
236 register_node(opq, outer_loop, entry, dom_depth(entry)); |
240 register_node(opq, outer_loop, entry, dom_depth(entry)); |
245 register_node(ifslow, outer_loop, iff, dom_depth(iff)); |
249 register_node(ifslow, outer_loop, iff, dom_depth(iff)); |
246 |
250 |
247 // Clone the loop body. The clone becomes the fast loop. The |
251 // Clone the loop body. The clone becomes the fast loop. The |
248 // original pre-header will (illegally) have 3 control users |
252 // original pre-header will (illegally) have 3 control users |
249 // (old & new loops & new if). |
253 // (old & new loops & new if). |
250 clone_loop(loop, old_new, dom_depth(head), iff); |
254 clone_loop(loop, old_new, dom_depth(head->skip_strip_mined()), mode, iff); |
251 assert(old_new[head->_idx]->is_Loop(), "" ); |
255 assert(old_new[head->_idx]->is_Loop(), "" ); |
252 |
256 |
253 // Fast (true) control |
257 // Fast (true) control |
254 Node* iffast_pred = clone_loop_predicates(entry, iffast, !counted_loop); |
258 Node* iffast_pred = clone_loop_predicates(entry, iffast, !counted_loop); |
255 _igvn.replace_input_of(head, LoopNode::EntryControl, iffast_pred); |
|
256 set_idom(head, iffast_pred, dom_depth(head)); |
|
257 |
259 |
258 // Slow (false) control |
260 // Slow (false) control |
259 Node* ifslow_pred = clone_loop_predicates(entry, ifslow, !counted_loop); |
261 Node* ifslow_pred = clone_loop_predicates(entry, ifslow, !counted_loop); |
260 LoopNode* slow_head = old_new[head->_idx]->as_Loop(); |
262 |
261 _igvn.replace_input_of(slow_head, LoopNode::EntryControl, ifslow_pred); |
263 Node* l = head->skip_strip_mined(); |
262 set_idom(slow_head, ifslow_pred, dom_depth(slow_head)); |
264 _igvn.replace_input_of(l, LoopNode::EntryControl, iffast_pred); |
|
265 set_idom(l, iffast_pred, dom_depth(l)); |
|
266 LoopNode* slow_l = old_new[head->_idx]->as_Loop()->skip_strip_mined(); |
|
267 _igvn.replace_input_of(slow_l, LoopNode::EntryControl, ifslow_pred); |
|
268 set_idom(slow_l, ifslow_pred, dom_depth(l)); |
263 |
269 |
264 recompute_dom_depth(); |
270 recompute_dom_depth(); |
265 |
271 |
266 return iffast; |
272 return iffast; |
267 } |
273 } |
268 |
274 |
269 LoopNode* PhaseIdealLoop::create_reserve_version_of_loop(IdealLoopTree *loop, CountedLoopReserveKit* lk) { |
275 LoopNode* PhaseIdealLoop::create_reserve_version_of_loop(IdealLoopTree *loop, CountedLoopReserveKit* lk) { |
270 Node_List old_new; |
276 Node_List old_new; |
271 LoopNode* head = loop->_head->as_Loop(); |
277 LoopNode* head = loop->_head->as_Loop(); |
272 bool counted_loop = head->is_CountedLoop(); |
278 bool counted_loop = head->is_CountedLoop(); |
273 Node* entry = head->in(LoopNode::EntryControl); |
279 Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl); |
274 _igvn.rehash_node_delayed(entry); |
280 _igvn.rehash_node_delayed(entry); |
275 IdealLoopTree* outer_loop = loop->_parent; |
281 IdealLoopTree* outer_loop = head->is_strip_mined() ? loop->_parent->_parent : loop->_parent; |
276 |
282 |
277 ConINode* const_1 = _igvn.intcon(1); |
283 ConINode* const_1 = _igvn.intcon(1); |
278 set_ctrl(const_1, C->root()); |
284 set_ctrl(const_1, C->root()); |
279 IfNode* iff = new IfNode(entry, const_1, PROB_MAX, COUNT_UNKNOWN); |
285 IfNode* iff = new IfNode(entry, const_1, PROB_MAX, COUNT_UNKNOWN); |
280 register_node(iff, outer_loop, entry, dom_depth(entry)); |
286 register_node(iff, outer_loop, entry, dom_depth(entry)); |
284 register_node(ifslow, outer_loop, iff, dom_depth(iff)); |
290 register_node(ifslow, outer_loop, iff, dom_depth(iff)); |
285 |
291 |
286 // Clone the loop body. The clone becomes the fast loop. The |
292 // Clone the loop body. The clone becomes the fast loop. The |
287 // original pre-header will (illegally) have 3 control users |
293 // original pre-header will (illegally) have 3 control users |
288 // (old & new loops & new if). |
294 // (old & new loops & new if). |
289 clone_loop(loop, old_new, dom_depth(head), iff); |
295 clone_loop(loop, old_new, dom_depth(head), CloneIncludesStripMined, iff); |
290 assert(old_new[head->_idx]->is_Loop(), "" ); |
296 assert(old_new[head->_idx]->is_Loop(), "" ); |
291 |
297 |
292 LoopNode* slow_head = old_new[head->_idx]->as_Loop(); |
298 LoopNode* slow_head = old_new[head->_idx]->as_Loop(); |
293 |
299 |
294 #ifndef PRODUCT |
300 #ifndef PRODUCT |
301 tty->print("\t before replace_input_of: slow_head = %d, ", slow_head->_idx); slow_head->dump(); |
307 tty->print("\t before replace_input_of: slow_head = %d, ", slow_head->_idx); slow_head->dump(); |
302 } |
308 } |
303 #endif |
309 #endif |
304 |
310 |
305 // Fast (true) control |
311 // Fast (true) control |
306 _igvn.replace_input_of(head, LoopNode::EntryControl, iffast); |
312 _igvn.replace_input_of(head->skip_strip_mined(), LoopNode::EntryControl, iffast); |
307 // Slow (false) control |
313 // Slow (false) control |
308 _igvn.replace_input_of(slow_head, LoopNode::EntryControl, ifslow); |
314 _igvn.replace_input_of(slow_head->skip_strip_mined(), LoopNode::EntryControl, ifslow); |
309 |
315 |
310 recompute_dom_depth(); |
316 recompute_dom_depth(); |
311 |
317 |
312 lk->set_iff(iff); |
318 lk->set_iff(iff); |
313 |
319 |