36 #include "opto/movenode.hpp" |
36 #include "opto/movenode.hpp" |
37 #include "opto/opaquenode.hpp" |
37 #include "opto/opaquenode.hpp" |
38 #include "opto/rootnode.hpp" |
38 #include "opto/rootnode.hpp" |
39 #include "opto/runtime.hpp" |
39 #include "opto/runtime.hpp" |
40 #include "opto/subnode.hpp" |
40 #include "opto/subnode.hpp" |
|
41 #include "opto/superword.hpp" |
41 #include "opto/vectornode.hpp" |
42 #include "opto/vectornode.hpp" |
42 |
43 |
43 //------------------------------is_loop_exit----------------------------------- |
44 //------------------------------is_loop_exit----------------------------------- |
44 // Given an IfNode, return the loop-exiting projection or NULL if both |
45 // Given an IfNode, return the loop-exiting projection or NULL if both |
45 // arms remain in the loop. |
46 // arms remain in the loop. |
638 |
639 |
639 |
640 |
640 //------------------------------policy_unroll---------------------------------- |
641 //------------------------------policy_unroll---------------------------------- |
641 // Return TRUE or FALSE if the loop should be unrolled or not. Unroll if |
642 // Return TRUE or FALSE if the loop should be unrolled or not. Unroll if |
642 // the loop is a CountedLoop and the body is small enough. |
643 // the loop is a CountedLoop and the body is small enough. |
643 bool IdealLoopTree::policy_unroll( PhaseIdealLoop *phase ) const { |
644 bool IdealLoopTree::policy_unroll(PhaseIdealLoop *phase) { |
644 |
645 |
645 CountedLoopNode *cl = _head->as_CountedLoop(); |
646 CountedLoopNode *cl = _head->as_CountedLoop(); |
646 assert(cl->is_normal_loop() || cl->is_main_loop(), ""); |
647 assert(cl->is_normal_loop() || cl->is_main_loop(), ""); |
647 |
648 |
648 if (!cl->is_valid_counted_loop()) |
649 if (!cl->is_valid_counted_loop()) |
650 |
651 |
651 // Protect against over-unrolling. |
652 // Protect against over-unrolling. |
652 // After split at least one iteration will be executed in pre-loop. |
653 // After split at least one iteration will be executed in pre-loop. |
653 if (cl->trip_count() <= (uint)(cl->is_normal_loop() ? 2 : 1)) return false; |
654 if (cl->trip_count() <= (uint)(cl->is_normal_loop() ? 2 : 1)) return false; |
654 |
655 |
|
656 _local_loop_unroll_limit = LoopUnrollLimit; |
|
657 _local_loop_unroll_factor = 4; |
655 int future_unroll_ct = cl->unrolled_count() * 2; |
658 int future_unroll_ct = cl->unrolled_count() * 2; |
656 if (future_unroll_ct > LoopMaxUnroll) return false; |
659 if (future_unroll_ct > LoopMaxUnroll) return false; |
657 |
660 |
658 // Check for initial stride being a small enough constant |
661 // Check for initial stride being a small enough constant |
659 if (abs(cl->stride_con()) > (1<<2)*future_unroll_ct) return false; |
662 if (abs(cl->stride_con()) > (1<<2)*future_unroll_ct) return false; |
745 } |
748 } |
746 #endif |
749 #endif |
747 } // switch |
750 } // switch |
748 } |
751 } |
749 |
752 |
|
753 if (UseSuperWord) { |
|
754 if (!cl->is_reduction_loop()) { |
|
755 phase->mark_reductions(this); |
|
756 } |
|
757 |
|
758 // Only attempt slp analysis when user controls do not prohibit it |
|
759 if (LoopMaxUnroll > _local_loop_unroll_factor) { |
|
760 // Once policy_slp_analysis succeeds, mark the loop with the |
|
761 // maximal unroll factor so that we minimize analysis passes |
|
762 if ((future_unroll_ct > _local_loop_unroll_factor) || |
|
763 (body_size > (uint)_local_loop_unroll_limit)) { |
|
764 policy_unroll_slp_analysis(cl, phase, future_unroll_ct); |
|
765 } |
|
766 } |
|
767 } |
|
768 |
750 // Check for being too big |
769 // Check for being too big |
751 if (body_size > (uint)LoopUnrollLimit) { |
770 if (body_size > (uint)_local_loop_unroll_limit) { |
752 if (xors_in_loop >= 4 && body_size < (uint)LoopUnrollLimit*4) return true; |
771 if (xors_in_loop >= 4 && body_size < (uint)LoopUnrollLimit*4) return true; |
753 // Normal case: loop too big |
772 // Normal case: loop too big |
754 return false; |
773 return false; |
755 } |
774 } |
756 |
775 |
757 // Unroll once! (Each trip will soon do double iterations) |
776 // Unroll once! (Each trip will soon do double iterations) |
758 return true; |
777 return true; |
|
778 } |
|
779 |
|
780 void IdealLoopTree::policy_unroll_slp_analysis(CountedLoopNode *cl, PhaseIdealLoop *phase, int future_unroll_ct) { |
|
781 // Enable this functionality target by target as needed |
|
782 if (SuperWordLoopUnrollAnalysis) { |
|
783 if (!cl->has_passed_slp()) { |
|
784 SuperWord sw(phase); |
|
785 sw.transform_loop(this, false); |
|
786 |
|
787 // If the loop is slp canonical analyze it |
|
788 if (sw.early_return() == false) { |
|
789 sw.unrolling_analysis(cl, _local_loop_unroll_factor); |
|
790 } |
|
791 } |
|
792 |
|
793 int slp_max_unroll_factor = cl->slp_max_unroll(); |
|
794 if ((slp_max_unroll_factor > 4) && |
|
795 (slp_max_unroll_factor >= future_unroll_ct)) { |
|
796 int new_limit = cl->node_count_before_unroll() * slp_max_unroll_factor; |
|
797 if (new_limit > LoopUnrollLimit) { |
|
798 #ifndef PRODUCT |
|
799 if (TraceSuperWordLoopUnrollAnalysis) { |
|
800 tty->print_cr("slp analysis is applying unroll limit %d, the original limit was %d\n", |
|
801 new_limit, _local_loop_unroll_limit); |
|
802 } |
|
803 #endif |
|
804 _local_loop_unroll_limit = new_limit; |
|
805 } |
|
806 } |
|
807 } |
759 } |
808 } |
760 |
809 |
761 //------------------------------policy_align----------------------------------- |
810 //------------------------------policy_align----------------------------------- |
762 // Return TRUE or FALSE if the loop should be cache-line aligned. Gather the |
811 // Return TRUE or FALSE if the loop should be cache-line aligned. Gather the |
763 // expression that does the alignment. Note that only one array base can be |
812 // expression that does the alignment. Note that only one array base can be |
2515 // twice as many iterations as before) and the main body limit (only do |
2565 // twice as many iterations as before) and the main body limit (only do |
2516 // an even number of trips). If we are peeling, we might enable some RCE |
2566 // an even number of trips). If we are peeling, we might enable some RCE |
2517 // and we'd rather unroll the post-RCE'd loop SO... do not unroll if |
2567 // and we'd rather unroll the post-RCE'd loop SO... do not unroll if |
2518 // peeling. |
2568 // peeling. |
2519 if (should_unroll && !should_peel) { |
2569 if (should_unroll && !should_peel) { |
2520 phase->mark_reductions(this); |
|
2521 phase->do_unroll(this, old_new, true); |
2570 phase->do_unroll(this, old_new, true); |
2522 } |
2571 } |
2523 |
2572 |
2524 // Adjust the pre-loop limits to align the main body |
2573 // Adjust the pre-loop limits to align the main body |
2525 // iterations. |
2574 // iterations. |