402 // These are chosen immediately. Some instructions are required to immediately |
402 // These are chosen immediately. Some instructions are required to immediately |
403 // precede the last instruction in the block, and these are taken last. Of the |
403 // precede the last instruction in the block, and these are taken last. Of the |
404 // remaining cases (most), choose the instruction with the greatest latency |
404 // remaining cases (most), choose the instruction with the greatest latency |
405 // (that is, the most number of pseudo-cycles required to the end of the |
405 // (that is, the most number of pseudo-cycles required to the end of the |
406 // routine). If there is a tie, choose the instruction with the most inputs. |
406 // routine). If there is a tie, choose the instruction with the most inputs. |
407 Node *Block::select(PhaseCFG *cfg, Node_List &worklist, int *ready_cnt, VectorSet &next_call, uint sched_slot) { |
407 Node *Block::select(PhaseCFG *cfg, Node_List &worklist, GrowableArray<int> &ready_cnt, VectorSet &next_call, uint sched_slot) { |
408 |
408 |
409 // If only a single entry on the stack, use it |
409 // If only a single entry on the stack, use it |
410 uint cnt = worklist.size(); |
410 uint cnt = worklist.size(); |
411 if (cnt == 1) { |
411 if (cnt == 1) { |
412 Node *n = worklist[0]; |
412 Node *n = worklist[0]; |
463 break; |
463 break; |
464 } |
464 } |
465 |
465 |
466 // More than this instruction pending for successor to be ready, |
466 // More than this instruction pending for successor to be ready, |
467 // don't choose this if other opportunities are ready |
467 // don't choose this if other opportunities are ready |
468 if (ready_cnt[use->_idx] > 1) |
468 if (ready_cnt.at(use->_idx) > 1) |
469 n_choice = 1; |
469 n_choice = 1; |
470 } |
470 } |
471 |
471 |
472 // loop terminated, prefer not to use this instruction |
472 // loop terminated, prefer not to use this instruction |
473 if (found_machif) |
473 if (found_machif) |
563 } |
563 } |
564 } |
564 } |
565 |
565 |
566 |
566 |
567 //------------------------------sched_call------------------------------------- |
567 //------------------------------sched_call------------------------------------- |
568 uint Block::sched_call( Matcher &matcher, Block_Array &bbs, uint node_cnt, Node_List &worklist, int *ready_cnt, MachCallNode *mcall, VectorSet &next_call ) { |
568 uint Block::sched_call( Matcher &matcher, Block_Array &bbs, uint node_cnt, Node_List &worklist, GrowableArray<int> &ready_cnt, MachCallNode *mcall, VectorSet &next_call ) { |
569 RegMask regs; |
569 RegMask regs; |
570 |
570 |
571 // Schedule all the users of the call right now. All the users are |
571 // Schedule all the users of the call right now. All the users are |
572 // projection Nodes, so they must be scheduled next to the call. |
572 // projection Nodes, so they must be scheduled next to the call. |
573 // Collect all the defined registers. |
573 // Collect all the defined registers. |
574 for (DUIterator_Fast imax, i = mcall->fast_outs(imax); i < imax; i++) { |
574 for (DUIterator_Fast imax, i = mcall->fast_outs(imax); i < imax; i++) { |
575 Node* n = mcall->fast_out(i); |
575 Node* n = mcall->fast_out(i); |
576 assert( n->is_MachProj(), "" ); |
576 assert( n->is_MachProj(), "" ); |
577 --ready_cnt[n->_idx]; |
577 int n_cnt = ready_cnt.at(n->_idx)-1; |
578 assert( !ready_cnt[n->_idx], "" ); |
578 ready_cnt.at_put(n->_idx, n_cnt); |
|
579 assert( n_cnt == 0, "" ); |
579 // Schedule next to call |
580 // Schedule next to call |
580 _nodes.map(node_cnt++, n); |
581 _nodes.map(node_cnt++, n); |
581 // Collect defined registers |
582 // Collect defined registers |
582 regs.OR(n->out_RegMask()); |
583 regs.OR(n->out_RegMask()); |
583 // Check for scheduling the next control-definer |
584 // Check for scheduling the next control-definer |
588 // Children of projections are now all ready |
589 // Children of projections are now all ready |
589 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) { |
590 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) { |
590 Node* m = n->fast_out(j); // Get user |
591 Node* m = n->fast_out(j); // Get user |
591 if( bbs[m->_idx] != this ) continue; |
592 if( bbs[m->_idx] != this ) continue; |
592 if( m->is_Phi() ) continue; |
593 if( m->is_Phi() ) continue; |
593 if( !--ready_cnt[m->_idx] ) |
594 int m_cnt = ready_cnt.at(m->_idx)-1; |
|
595 ready_cnt.at_put(m->_idx, m_cnt); |
|
596 if( m_cnt == 0 ) |
594 worklist.push(m); |
597 worklist.push(m); |
595 } |
598 } |
596 |
599 |
597 } |
600 } |
598 |
601 |
653 } |
656 } |
654 |
657 |
655 |
658 |
656 //------------------------------schedule_local--------------------------------- |
659 //------------------------------schedule_local--------------------------------- |
657 // Topological sort within a block. Someday become a real scheduler. |
660 // Topological sort within a block. Someday become a real scheduler. |
658 bool Block::schedule_local(PhaseCFG *cfg, Matcher &matcher, int *ready_cnt, VectorSet &next_call) { |
661 bool Block::schedule_local(PhaseCFG *cfg, Matcher &matcher, GrowableArray<int> &ready_cnt, VectorSet &next_call) { |
659 // Already "sorted" are the block start Node (as the first entry), and |
662 // Already "sorted" are the block start Node (as the first entry), and |
660 // the block-ending Node and any trailing control projections. We leave |
663 // the block-ending Node and any trailing control projections. We leave |
661 // these alone. PhiNodes and ParmNodes are made to follow the block start |
664 // these alone. PhiNodes and ParmNodes are made to follow the block start |
662 // Node. Everything else gets topo-sorted. |
665 // Node. Everything else gets topo-sorted. |
663 |
666 |
693 for( uint j=0; j<cnt; j++ ) { |
696 for( uint j=0; j<cnt; j++ ) { |
694 Node *m = n->in(j); |
697 Node *m = n->in(j); |
695 if( m && cfg->_bbs[m->_idx] == this && !m->is_top() ) |
698 if( m && cfg->_bbs[m->_idx] == this && !m->is_top() ) |
696 local++; // One more block-local input |
699 local++; // One more block-local input |
697 } |
700 } |
698 ready_cnt[n->_idx] = local; // Count em up |
701 ready_cnt.at_put(n->_idx, local); // Count em up |
699 |
702 |
700 #ifdef ASSERT |
703 #ifdef ASSERT |
701 if( UseConcMarkSweepGC || UseG1GC ) { |
704 if( UseConcMarkSweepGC || UseG1GC ) { |
702 if( n->is_Mach() && n->as_Mach()->ideal_Opcode() == Op_StoreCM ) { |
705 if( n->is_Mach() && n->as_Mach()->ideal_Opcode() == Op_StoreCM ) { |
703 // Check the precedence edges |
706 // Check the precedence edges |
727 n->add_prec(x); |
730 n->add_prec(x); |
728 } |
731 } |
729 } |
732 } |
730 } |
733 } |
731 for(uint i2=i; i2<_nodes.size(); i2++ ) // Trailing guys get zapped count |
734 for(uint i2=i; i2<_nodes.size(); i2++ ) // Trailing guys get zapped count |
732 ready_cnt[_nodes[i2]->_idx] = 0; |
735 ready_cnt.at_put(_nodes[i2]->_idx, 0); |
733 |
736 |
734 // All the prescheduled guys do not hold back internal nodes |
737 // All the prescheduled guys do not hold back internal nodes |
735 uint i3; |
738 uint i3; |
736 for(i3 = 0; i3<phi_cnt; i3++ ) { // For all pre-scheduled |
739 for(i3 = 0; i3<phi_cnt; i3++ ) { // For all pre-scheduled |
737 Node *n = _nodes[i3]; // Get pre-scheduled |
740 Node *n = _nodes[i3]; // Get pre-scheduled |
738 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) { |
741 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) { |
739 Node* m = n->fast_out(j); |
742 Node* m = n->fast_out(j); |
740 if( cfg->_bbs[m->_idx] ==this ) // Local-block user |
743 if( cfg->_bbs[m->_idx] ==this ) { // Local-block user |
741 ready_cnt[m->_idx]--; // Fix ready count |
744 int m_cnt = ready_cnt.at(m->_idx)-1; |
|
745 ready_cnt.at_put(m->_idx, m_cnt); // Fix ready count |
|
746 } |
742 } |
747 } |
743 } |
748 } |
744 |
749 |
745 Node_List delay; |
750 Node_List delay; |
746 // Make a worklist |
751 // Make a worklist |
747 Node_List worklist; |
752 Node_List worklist; |
748 for(uint i4=i3; i4<node_cnt; i4++ ) { // Put ready guys on worklist |
753 for(uint i4=i3; i4<node_cnt; i4++ ) { // Put ready guys on worklist |
749 Node *m = _nodes[i4]; |
754 Node *m = _nodes[i4]; |
750 if( !ready_cnt[m->_idx] ) { // Zero ready count? |
755 if( !ready_cnt.at(m->_idx) ) { // Zero ready count? |
751 if (m->is_iteratively_computed()) { |
756 if (m->is_iteratively_computed()) { |
752 // Push induction variable increments last to allow other uses |
757 // Push induction variable increments last to allow other uses |
753 // of the phi to be scheduled first. The select() method breaks |
758 // of the phi to be scheduled first. The select() method breaks |
754 // ties in scheduling by worklist order. |
759 // ties in scheduling by worklist order. |
755 delay.push(m); |
760 delay.push(m); |
773 #ifndef PRODUCT |
778 #ifndef PRODUCT |
774 if (cfg->trace_opto_pipelining()) { |
779 if (cfg->trace_opto_pipelining()) { |
775 for (uint j=0; j<_nodes.size(); j++) { |
780 for (uint j=0; j<_nodes.size(); j++) { |
776 Node *n = _nodes[j]; |
781 Node *n = _nodes[j]; |
777 int idx = n->_idx; |
782 int idx = n->_idx; |
778 tty->print("# ready cnt:%3d ", ready_cnt[idx]); |
783 tty->print("# ready cnt:%3d ", ready_cnt.at(idx)); |
779 tty->print("latency:%3d ", cfg->_node_latency->at_grow(idx)); |
784 tty->print("latency:%3d ", cfg->_node_latency->at_grow(idx)); |
780 tty->print("%4d: %s\n", idx, n->Name()); |
785 tty->print("%4d: %s\n", idx, n->Name()); |
781 } |
786 } |
782 } |
787 } |
783 #endif |
788 #endif |
784 |
789 |
785 uint max_idx = matcher.C->unique(); |
790 uint max_idx = (uint)ready_cnt.length(); |
786 // Pull from worklist and schedule |
791 // Pull from worklist and schedule |
787 while( worklist.size() ) { // Worklist is not ready |
792 while( worklist.size() ) { // Worklist is not ready |
788 |
793 |
789 #ifndef PRODUCT |
794 #ifndef PRODUCT |
790 if (cfg->trace_opto_pipelining()) { |
795 if (cfg->trace_opto_pipelining()) { |
838 // Children are now all ready |
843 // Children are now all ready |
839 for (DUIterator_Fast i5max, i5 = n->fast_outs(i5max); i5 < i5max; i5++) { |
844 for (DUIterator_Fast i5max, i5 = n->fast_outs(i5max); i5 < i5max; i5++) { |
840 Node* m = n->fast_out(i5); // Get user |
845 Node* m = n->fast_out(i5); // Get user |
841 if( cfg->_bbs[m->_idx] != this ) continue; |
846 if( cfg->_bbs[m->_idx] != this ) continue; |
842 if( m->is_Phi() ) continue; |
847 if( m->is_Phi() ) continue; |
843 if (m->_idx > max_idx) { // new node, skip it |
848 if (m->_idx >= max_idx) { // new node, skip it |
844 assert(m->is_MachProj() && n->is_Mach() && n->as_Mach()->has_call(), "unexpected node types"); |
849 assert(m->is_MachProj() && n->is_Mach() && n->as_Mach()->has_call(), "unexpected node types"); |
845 continue; |
850 continue; |
846 } |
851 } |
847 if( !--ready_cnt[m->_idx] ) |
852 int m_cnt = ready_cnt.at(m->_idx)-1; |
|
853 ready_cnt.at_put(m->_idx, m_cnt); |
|
854 if( m_cnt == 0 ) |
848 worklist.push(m); |
855 worklist.push(m); |
849 } |
856 } |
850 } |
857 } |
851 |
858 |
852 if( phi_cnt != end_idx() ) { |
859 if( phi_cnt != end_idx() ) { |