942 //remove "current" from its current position in the memory graph and insert |
942 //remove "current" from its current position in the memory graph and insert |
943 //it after the appropriate insertion point (lip or uip) |
943 //it after the appropriate insertion point (lip or uip) |
944 void SuperWord::remove_and_insert(MemNode *current, MemNode *prev, MemNode *lip, |
944 void SuperWord::remove_and_insert(MemNode *current, MemNode *prev, MemNode *lip, |
945 Node *uip, Unique_Node_List &sched_before) { |
945 Node *uip, Unique_Node_List &sched_before) { |
946 Node* my_mem = current->in(MemNode::Memory); |
946 Node* my_mem = current->in(MemNode::Memory); |
947 _igvn.hash_delete(current); |
947 _igvn.rehash_node_delayed(current); |
948 _igvn.hash_delete(my_mem); |
948 _igvn.hash_delete(my_mem); |
949 |
949 |
950 //remove current_store from its current position in the memmory graph |
950 //remove current_store from its current position in the memmory graph |
951 for (DUIterator i = current->outs(); current->has_out(i); i++) { |
951 for (DUIterator i = current->outs(); current->has_out(i); i++) { |
952 Node* use = current->out(i); |
952 Node* use = current->out(i); |
953 if (use->is_Mem()) { |
953 if (use->is_Mem()) { |
954 assert(use->in(MemNode::Memory) == current, "must be"); |
954 assert(use->in(MemNode::Memory) == current, "must be"); |
955 _igvn.hash_delete(use); |
955 _igvn.rehash_node_delayed(use); |
956 if (use == prev) { // connect prev to my_mem |
956 if (use == prev) { // connect prev to my_mem |
957 use->set_req(MemNode::Memory, my_mem); |
957 use->set_req(MemNode::Memory, my_mem); |
958 } else if (sched_before.member(use)) { |
958 } else if (sched_before.member(use)) { |
959 _igvn.hash_delete(uip); |
959 _igvn.hash_delete(uip); |
960 use->set_req(MemNode::Memory, uip); |
960 use->set_req(MemNode::Memory, uip); |
961 } else { |
961 } else { |
962 _igvn.hash_delete(lip); |
962 _igvn.hash_delete(lip); |
963 use->set_req(MemNode::Memory, lip); |
963 use->set_req(MemNode::Memory, lip); |
964 } |
964 } |
965 _igvn._worklist.push(use); |
|
966 --i; //deleted this edge; rescan position |
965 --i; //deleted this edge; rescan position |
967 } |
966 } |
968 } |
967 } |
969 |
968 |
970 bool sched_up = sched_before.member(current); |
969 bool sched_up = sched_before.member(current); |
974 // all uses of insert_pt's memory state should use current's instead |
973 // all uses of insert_pt's memory state should use current's instead |
975 for (DUIterator i = insert_pt->outs(); insert_pt->has_out(i); i++) { |
974 for (DUIterator i = insert_pt->outs(); insert_pt->has_out(i); i++) { |
976 Node* use = insert_pt->out(i); |
975 Node* use = insert_pt->out(i); |
977 if (use->is_Mem()) { |
976 if (use->is_Mem()) { |
978 assert(use->in(MemNode::Memory) == insert_pt, "must be"); |
977 assert(use->in(MemNode::Memory) == insert_pt, "must be"); |
979 _igvn.hash_delete(use); |
978 _igvn.replace_input_of(use, MemNode::Memory, current); |
980 use->set_req(MemNode::Memory, current); |
|
981 _igvn._worklist.push(use); |
|
982 --i; //deleted this edge; rescan position |
979 --i; //deleted this edge; rescan position |
983 } else if (!sched_up && use->is_Phi() && use->bottom_type() == Type::MEMORY) { |
980 } else if (!sched_up && use->is_Phi() && use->bottom_type() == Type::MEMORY) { |
984 uint pos; //lip (lower insert point) must be the last one in the memory slice |
981 uint pos; //lip (lower insert point) must be the last one in the memory slice |
985 _igvn.hash_delete(use); |
|
986 for (pos=1; pos < use->req(); pos++) { |
982 for (pos=1; pos < use->req(); pos++) { |
987 if (use->in(pos) == insert_pt) break; |
983 if (use->in(pos) == insert_pt) break; |
988 } |
984 } |
989 use->set_req(pos, current); |
985 _igvn.replace_input_of(use, pos, current); |
990 _igvn._worklist.push(use); |
|
991 --i; |
986 --i; |
992 } |
987 } |
993 } |
988 } |
994 |
989 |
995 //connect current to insert_pt |
990 //connect current to insert_pt |
996 current->set_req(MemNode::Memory, insert_pt); |
991 current->set_req(MemNode::Memory, insert_pt); |
997 _igvn._worklist.push(current); |
|
998 } |
992 } |
999 |
993 |
1000 //------------------------------co_locate_pack---------------------------------- |
994 //------------------------------co_locate_pack---------------------------------- |
1001 // To schedule a store pack, we need to move any sandwiched memory ops either before |
995 // To schedule a store pack, we need to move any sandwiched memory ops either before |
1002 // or after the pack, based upon dependence information: |
996 // or after the pack, based upon dependence information: |
1075 _igvn.hash_delete(current); |
1069 _igvn.hash_delete(current); |
1076 for (DUIterator i = current->outs(); current->has_out(i); i++) { |
1070 for (DUIterator i = current->outs(); current->has_out(i); i++) { |
1077 Node* use = current->out(i); |
1071 Node* use = current->out(i); |
1078 if (use->is_Mem() && use != previous) { |
1072 if (use->is_Mem() && use != previous) { |
1079 assert(use->in(MemNode::Memory) == current, "must be"); |
1073 assert(use->in(MemNode::Memory) == current, "must be"); |
1080 _igvn.hash_delete(use); |
|
1081 if (schedule_before_pack.member(use)) { |
1074 if (schedule_before_pack.member(use)) { |
1082 _igvn.hash_delete(upper_insert_pt); |
1075 _igvn.hash_delete(upper_insert_pt); |
1083 use->set_req(MemNode::Memory, upper_insert_pt); |
1076 _igvn.replace_input_of(use, MemNode::Memory, upper_insert_pt); |
1084 } else { |
1077 } else { |
1085 _igvn.hash_delete(lower_insert_pt); |
1078 _igvn.hash_delete(lower_insert_pt); |
1086 use->set_req(MemNode::Memory, lower_insert_pt); |
1079 _igvn.replace_input_of(use, MemNode::Memory, lower_insert_pt); |
1087 } |
1080 } |
1088 _igvn._worklist.push(use); |
|
1089 --i; // deleted this edge; rescan position |
1081 --i; // deleted this edge; rescan position |
1090 } |
1082 } |
1091 } |
1083 } |
1092 previous = current; |
1084 previous = current; |
1093 } else { // !in_pack(current, pk) ==> a sandwiched store |
1085 } else { // !in_pack(current, pk) ==> a sandwiched store |
1120 Node* mem_input = schedule_last ? last_mem : first_mem; |
1112 Node* mem_input = schedule_last ? last_mem : first_mem; |
1121 _igvn.hash_delete(mem_input); |
1113 _igvn.hash_delete(mem_input); |
1122 // Give each load the same memory state |
1114 // Give each load the same memory state |
1123 for (uint i = 0; i < pk->size(); i++) { |
1115 for (uint i = 0; i < pk->size(); i++) { |
1124 LoadNode* ld = pk->at(i)->as_Load(); |
1116 LoadNode* ld = pk->at(i)->as_Load(); |
1125 _igvn.hash_delete(ld); |
1117 _igvn.replace_input_of(ld, MemNode::Memory, mem_input); |
1126 ld->set_req(MemNode::Memory, mem_input); |
|
1127 _igvn._worklist.push(ld); |
|
1128 } |
1118 } |
1129 } |
1119 } |
1130 } |
1120 } |
1131 |
1121 |
1132 //------------------------------output--------------------------- |
1122 //------------------------------output--------------------------- |
1280 _n_idx_list.pop(); |
1270 _n_idx_list.pop(); |
1281 Node* def = use->in(idx); |
1271 Node* def = use->in(idx); |
1282 |
1272 |
1283 // Insert extract operation |
1273 // Insert extract operation |
1284 _igvn.hash_delete(def); |
1274 _igvn.hash_delete(def); |
1285 _igvn.hash_delete(use); |
|
1286 int def_pos = alignment(def) / data_size(def); |
1275 int def_pos = alignment(def) / data_size(def); |
1287 const Type* def_t = velt_type(def); |
1276 const Type* def_t = velt_type(def); |
1288 |
1277 |
1289 Node* ex = ExtractNode::make(_phase->C, def, def_pos, def_t); |
1278 Node* ex = ExtractNode::make(_phase->C, def, def_pos, def_t); |
1290 _phase->_igvn.register_new_node_with_optimizer(ex); |
1279 _phase->_igvn.register_new_node_with_optimizer(ex); |
1291 _phase->set_ctrl(ex, _phase->get_ctrl(def)); |
1280 _phase->set_ctrl(ex, _phase->get_ctrl(def)); |
1292 use->set_req(idx, ex); |
1281 _igvn.replace_input_of(use, idx, ex); |
1293 _igvn._worklist.push(def); |
1282 _igvn._worklist.push(def); |
1294 _igvn._worklist.push(use); |
|
1295 |
1283 |
1296 bb_insert_after(ex, bb_idx(def)); |
1284 bb_insert_after(ex, bb_idx(def)); |
1297 set_velt_type(ex, def_t); |
1285 set_velt_type(ex, def_t); |
1298 } |
1286 } |
1299 } |
1287 } |