67 HasReductions=128, |
67 HasReductions=128, |
68 WasSlpAnalyzed=256, |
68 WasSlpAnalyzed=256, |
69 PassedSlpAnalysis=512, |
69 PassedSlpAnalysis=512, |
70 DoUnrollOnly=1024, |
70 DoUnrollOnly=1024, |
71 VectorizedLoop=2048, |
71 VectorizedLoop=2048, |
72 HasAtomicPostLoop=4096 }; |
72 HasAtomicPostLoop=4096, |
|
73 HasRangeChecks=8192, |
|
74 IsMultiversioned=16384}; |
73 char _unswitch_count; |
75 char _unswitch_count; |
74 enum { _unswitch_max=3 }; |
76 enum { _unswitch_max=3 }; |
|
77 char _postloop_flags; |
|
78 enum { LoopNotRCEChecked = 0, LoopRCEChecked = 1, RCEPostLoop = 2 }; |
75 |
79 |
76 public: |
80 public: |
77 // Names for edge indices |
81 // Names for edge indices |
78 enum { Self=0, EntryControl, LoopBackControl }; |
82 enum { Self=0, EntryControl, LoopBackControl }; |
79 |
83 |
80 int is_inner_loop() const { return _loop_flags & InnerLoop; } |
84 int is_inner_loop() const { return _loop_flags & InnerLoop; } |
81 void set_inner_loop() { _loop_flags |= InnerLoop; } |
85 void set_inner_loop() { _loop_flags |= InnerLoop; } |
82 |
86 |
|
87 int range_checks_present() const { return _loop_flags & HasRangeChecks; } |
|
88 int is_multiversioned() const { return _loop_flags & IsMultiversioned; } |
|
89 int is_vectorized_loop() const { return _loop_flags & VectorizedLoop; } |
83 int is_partial_peel_loop() const { return _loop_flags & PartialPeelLoop; } |
90 int is_partial_peel_loop() const { return _loop_flags & PartialPeelLoop; } |
84 void set_partial_peel_loop() { _loop_flags |= PartialPeelLoop; } |
91 void set_partial_peel_loop() { _loop_flags |= PartialPeelLoop; } |
85 int partial_peel_has_failed() const { return _loop_flags & PartialPeelFailed; } |
92 int partial_peel_has_failed() const { return _loop_flags & PartialPeelFailed; } |
|
93 |
86 void mark_partial_peel_failed() { _loop_flags |= PartialPeelFailed; } |
94 void mark_partial_peel_failed() { _loop_flags |= PartialPeelFailed; } |
87 void mark_has_reductions() { _loop_flags |= HasReductions; } |
95 void mark_has_reductions() { _loop_flags |= HasReductions; } |
88 void mark_was_slp() { _loop_flags |= WasSlpAnalyzed; } |
96 void mark_was_slp() { _loop_flags |= WasSlpAnalyzed; } |
89 void mark_passed_slp() { _loop_flags |= PassedSlpAnalysis; } |
97 void mark_passed_slp() { _loop_flags |= PassedSlpAnalysis; } |
90 void mark_do_unroll_only() { _loop_flags |= DoUnrollOnly; } |
98 void mark_do_unroll_only() { _loop_flags |= DoUnrollOnly; } |
91 void mark_loop_vectorized() { _loop_flags |= VectorizedLoop; } |
99 void mark_loop_vectorized() { _loop_flags |= VectorizedLoop; } |
92 void mark_has_atomic_post_loop() { _loop_flags |= HasAtomicPostLoop; } |
100 void mark_has_atomic_post_loop() { _loop_flags |= HasAtomicPostLoop; } |
|
101 void mark_has_range_checks() { _loop_flags |= HasRangeChecks; } |
|
102 void mark_is_multiversioned() { _loop_flags |= IsMultiversioned; } |
93 |
103 |
94 int unswitch_max() { return _unswitch_max; } |
104 int unswitch_max() { return _unswitch_max; } |
95 int unswitch_count() { return _unswitch_count; } |
105 int unswitch_count() { return _unswitch_count; } |
|
106 |
|
107 int has_been_range_checked() const { return _postloop_flags & LoopRCEChecked; } |
|
108 void set_has_been_range_checked() { _postloop_flags |= LoopRCEChecked; } |
|
109 int is_rce_post_loop() const { return _postloop_flags & RCEPostLoop; } |
|
110 void set_is_rce_post_loop() { _postloop_flags |= RCEPostLoop; } |
|
111 |
96 void set_unswitch_count(int val) { |
112 void set_unswitch_count(int val) { |
97 assert (val <= unswitch_max(), "too many unswitches"); |
113 assert (val <= unswitch_max(), "too many unswitches"); |
98 _unswitch_count = val; |
114 _unswitch_count = val; |
99 } |
115 } |
100 |
116 |
101 LoopNode( Node *entry, Node *backedge ) : RegionNode(3), _loop_flags(0), _unswitch_count(0) { |
117 LoopNode(Node *entry, Node *backedge) : RegionNode(3), _loop_flags(0), _unswitch_count(0), _postloop_flags(0) { |
102 init_class_id(Class_Loop); |
118 init_class_id(Class_Loop); |
103 init_req(EntryControl, entry); |
119 init_req(EntryControl, entry); |
104 init_req(LoopBackControl, backedge); |
120 init_req(LoopBackControl, backedge); |
105 } |
121 } |
106 |
122 |
223 int is_reduction_loop() const { return (_loop_flags&HasReductions) == HasReductions; } |
239 int is_reduction_loop() const { return (_loop_flags&HasReductions) == HasReductions; } |
224 int was_slp_analyzed () const { return (_loop_flags&WasSlpAnalyzed) == WasSlpAnalyzed; } |
240 int was_slp_analyzed () const { return (_loop_flags&WasSlpAnalyzed) == WasSlpAnalyzed; } |
225 int has_passed_slp () const { return (_loop_flags&PassedSlpAnalysis) == PassedSlpAnalysis; } |
241 int has_passed_slp () const { return (_loop_flags&PassedSlpAnalysis) == PassedSlpAnalysis; } |
226 int do_unroll_only () const { return (_loop_flags&DoUnrollOnly) == DoUnrollOnly; } |
242 int do_unroll_only () const { return (_loop_flags&DoUnrollOnly) == DoUnrollOnly; } |
227 int is_main_no_pre_loop() const { return _loop_flags & MainHasNoPreLoop; } |
243 int is_main_no_pre_loop() const { return _loop_flags & MainHasNoPreLoop; } |
228 int is_vectorized_loop () const { return (_loop_flags & VectorizedLoop) == VectorizedLoop; } |
|
229 int has_atomic_post_loop () const { return (_loop_flags & HasAtomicPostLoop) == HasAtomicPostLoop; } |
244 int has_atomic_post_loop () const { return (_loop_flags & HasAtomicPostLoop) == HasAtomicPostLoop; } |
230 void set_main_no_pre_loop() { _loop_flags |= MainHasNoPreLoop; } |
245 void set_main_no_pre_loop() { _loop_flags |= MainHasNoPreLoop; } |
231 |
246 |
232 int main_idx() const { return _main_idx; } |
247 int main_idx() const { return _main_idx; } |
233 |
248 |
909 void do_peeling( IdealLoopTree *loop, Node_List &old_new ); |
924 void do_peeling( IdealLoopTree *loop, Node_List &old_new ); |
910 |
925 |
911 // Add pre and post loops around the given loop. These loops are used |
926 // Add pre and post loops around the given loop. These loops are used |
912 // during RCE, unrolling and aligning loops. |
927 // during RCE, unrolling and aligning loops. |
913 void insert_pre_post_loops( IdealLoopTree *loop, Node_List &old_new, bool peel_only ); |
928 void insert_pre_post_loops( IdealLoopTree *loop, Node_List &old_new, bool peel_only ); |
|
929 |
|
930 // Add post loop after the given loop. |
|
931 Node *insert_post_loop(IdealLoopTree *loop, Node_List &old_new, |
|
932 CountedLoopNode *main_head, CountedLoopEndNode *main_end, |
|
933 Node *incr, Node *limit, CountedLoopNode *&post_head); |
|
934 |
|
935 // Add an RCE'd post loop which we will multi-version adapt for run time test path usage |
|
936 void insert_scalar_rced_post_loop( IdealLoopTree *loop, Node_List &old_new ); |
|
937 |
914 // Add a vector post loop between a vector main loop and the current post loop |
938 // Add a vector post loop between a vector main loop and the current post loop |
915 void insert_vector_post_loop(IdealLoopTree *loop, Node_List &old_new); |
939 void insert_vector_post_loop(IdealLoopTree *loop, Node_List &old_new); |
916 // If Node n lives in the back_ctrl block, we clone a private version of n |
940 // If Node n lives in the back_ctrl block, we clone a private version of n |
917 // in preheader_ctrl block and return that, otherwise return n. |
941 // in preheader_ctrl block and return that, otherwise return n. |
918 Node *clone_up_backedge_goo( Node *back_ctrl, Node *preheader_ctrl, Node *n, VectorSet &visited, Node_Stack &clones ); |
942 Node *clone_up_backedge_goo( Node *back_ctrl, Node *preheader_ctrl, Node *n, VectorSet &visited, Node_Stack &clones ); |
981 bool is_node_unreachable(Node *n) const { |
1005 bool is_node_unreachable(Node *n) const { |
982 return !has_node(n) || n->is_unreachable(_igvn); |
1006 return !has_node(n) || n->is_unreachable(_igvn); |
983 } |
1007 } |
984 |
1008 |
985 // Eliminate range-checks and other trip-counter vs loop-invariant tests. |
1009 // Eliminate range-checks and other trip-counter vs loop-invariant tests. |
986 void do_range_check( IdealLoopTree *loop, Node_List &old_new ); |
1010 int do_range_check( IdealLoopTree *loop, Node_List &old_new ); |
|
1011 |
|
1012 // Check to see if do_range_check(...) cleaned the main loop of range-checks |
|
1013 void has_range_checks(IdealLoopTree *loop); |
|
1014 |
|
1015 // Process post loops which have range checks and try to build a multi-version |
|
1016 // guard to safely determine if we can execute the post loop which was RCE'd. |
|
1017 bool multi_version_post_loops(IdealLoopTree *rce_loop, IdealLoopTree *legacy_loop); |
|
1018 |
|
1019 // Cause the rce'd post loop to optimized away, this happens if we cannot complete multiverioning |
|
1020 void poison_rce_post_loop(IdealLoopTree *rce_loop); |
987 |
1021 |
988 // Create a slow version of the loop by cloning the loop |
1022 // Create a slow version of the loop by cloning the loop |
989 // and inserting an if to select fast-slow versions. |
1023 // and inserting an if to select fast-slow versions. |
990 ProjNode* create_slow_version_of_loop(IdealLoopTree *loop, |
1024 ProjNode* create_slow_version_of_loop(IdealLoopTree *loop, |
991 Node_List &old_new, |
1025 Node_List &old_new, |