988 // (4) If there is no dependence restriction for a sandwiched memory op, we simply |
988 // (4) If there is no dependence restriction for a sandwiched memory op, we simply |
989 // schedule this store AFTER the pack |
989 // schedule this store AFTER the pack |
990 // (5) We know there is no dependence cycle, so there in no other case; |
990 // (5) We know there is no dependence cycle, so there in no other case; |
991 // (6) Finally, all memory ops in another single pack should be moved in the same direction. |
991 // (6) Finally, all memory ops in another single pack should be moved in the same direction. |
992 // |
992 // |
993 // To schedule a load pack: the memory edge of every loads in the pack must be |
993 // To schedule a load pack, we use the memory state of either the first or the last load in |
994 // the same as the memory edge of the last executed load in the pack |
994 // the pack, based on the dependence constraint. |
995 void SuperWord::co_locate_pack(Node_List* pk) { |
995 void SuperWord::co_locate_pack(Node_List* pk) { |
996 if (pk->at(0)->is_Store()) { |
996 if (pk->at(0)->is_Store()) { |
997 MemNode* first = executed_first(pk)->as_Mem(); |
997 MemNode* first = executed_first(pk)->as_Mem(); |
998 MemNode* last = executed_last(pk)->as_Mem(); |
998 MemNode* last = executed_last(pk)->as_Mem(); |
999 Unique_Node_List schedule_before_pack; |
999 Unique_Node_List schedule_before_pack; |
1074 |
1074 |
1075 if (current == first) break; |
1075 if (current == first) break; |
1076 current = my_mem->as_Mem(); |
1076 current = my_mem->as_Mem(); |
1077 } // end while |
1077 } // end while |
1078 } else if (pk->at(0)->is_Load()) { //load |
1078 } else if (pk->at(0)->is_Load()) { //load |
1079 // all use the memory state that the last executed load uses |
1079 // all loads in the pack should have the same memory state. By default, |
1080 LoadNode* last_load = executed_last(pk)->as_Load(); |
1080 // we use the memory state of the last load. However, if any load could |
1081 Node* last_mem = last_load->in(MemNode::Memory); |
1081 // not be moved down due to the dependence constraint, we use the memory |
1082 _igvn.hash_delete(last_mem); |
1082 // state of the first load. |
1083 // Give each load same memory state as last |
1083 Node* last_mem = executed_last(pk)->in(MemNode::Memory); |
|
1084 Node* first_mem = executed_first(pk)->in(MemNode::Memory); |
|
1085 bool schedule_last = true; |
|
1086 for (uint i = 0; i < pk->size(); i++) { |
|
1087 Node* ld = pk->at(i); |
|
1088 for (Node* current = last_mem; current != ld->in(MemNode::Memory); |
|
1089 current=current->in(MemNode::Memory)) { |
|
1090 assert(current != first_mem, "corrupted memory graph"); |
|
1091 if(current->is_Mem() && !independent(current, ld)){ |
|
1092 schedule_last = false; // a later store depends on this load |
|
1093 break; |
|
1094 } |
|
1095 } |
|
1096 } |
|
1097 |
|
1098 Node* mem_input = schedule_last ? last_mem : first_mem; |
|
1099 _igvn.hash_delete(mem_input); |
|
1100 // Give each load the same memory state |
1084 for (uint i = 0; i < pk->size(); i++) { |
1101 for (uint i = 0; i < pk->size(); i++) { |
1085 LoadNode* ld = pk->at(i)->as_Load(); |
1102 LoadNode* ld = pk->at(i)->as_Load(); |
1086 _igvn.hash_delete(ld); |
1103 _igvn.hash_delete(ld); |
1087 ld->set_req(MemNode::Memory, last_mem); |
1104 ld->set_req(MemNode::Memory, mem_input); |
1088 _igvn._worklist.push(ld); |
1105 _igvn._worklist.push(ld); |
1089 } |
1106 } |
1090 } |
1107 } |
1091 } |
1108 } |
1092 |
1109 |