author | kvn |
Tue, 05 May 2015 12:33:57 -0700 | |
changeset 30593 | 69f942690128 |
parent 30588 | 24fc4b3a964e |
child 30625 | 80a08f9b2d63 |
permissions | -rw-r--r-- |
1 | 1 |
/* |
30593 | 2 |
* Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved. |
1 | 3 |
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
4 |
* |
|
5 |
* This code is free software; you can redistribute it and/or modify it |
|
6 |
* under the terms of the GNU General Public License version 2 only, as |
|
7 |
* published by the Free Software Foundation. |
|
8 |
* |
|
9 |
* This code is distributed in the hope that it will be useful, but WITHOUT |
|
10 |
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
|
11 |
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
|
12 |
* version 2 for more details (a copy is included in the LICENSE file that |
|
13 |
* accompanied this code). |
|
14 |
* |
|
15 |
* You should have received a copy of the GNU General Public License version |
|
16 |
* 2 along with this work; if not, write to the Free Software Foundation, |
|
17 |
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
|
18 |
* |
|
5547
f4b087cbb361
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
4428
diff
changeset
|
19 |
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
f4b087cbb361
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
4428
diff
changeset
|
20 |
* or visit www.oracle.com if you need additional information or have any |
f4b087cbb361
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
4428
diff
changeset
|
21 |
* questions. |
1 | 22 |
*/ |
23 |
||
7397 | 24 |
#include "precompiled.hpp" |
25 |
#include "compiler/compileLog.hpp" |
|
26 |
#include "libadt/vectset.hpp" |
|
27 |
#include "memory/allocation.inline.hpp" |
|
28 |
#include "opto/addnode.hpp" |
|
29 |
#include "opto/callnode.hpp" |
|
23528 | 30 |
#include "opto/castnode.hpp" |
31 |
#include "opto/convertnode.hpp" |
|
7397 | 32 |
#include "opto/divnode.hpp" |
33 |
#include "opto/matcher.hpp" |
|
34 |
#include "opto/memnode.hpp" |
|
35 |
#include "opto/mulnode.hpp" |
|
36 |
#include "opto/opcodes.hpp" |
|
23528 | 37 |
#include "opto/opaquenode.hpp" |
7397 | 38 |
#include "opto/superword.hpp" |
39 |
#include "opto/vectornode.hpp" |
|
1 | 40 |
|
41 |
// |
|
42 |
// S U P E R W O R D T R A N S F O R M |
|
43 |
//============================================================================= |
|
44 |
||
45 |
//------------------------------SuperWord--------------------------- |
|
46 |
SuperWord::SuperWord(PhaseIdealLoop* phase) : |
|
47 |
_phase(phase), |
|
48 |
_igvn(phase->_igvn), |
|
49 |
_arena(phase->C->comp_arena()), |
|
50 |
_packset(arena(), 8, 0, NULL), // packs for the current block |
|
51 |
_bb_idx(arena(), (int)(1.10 * phase->C->unique()), 0, 0), // node idx to index in bb |
|
52 |
_block(arena(), 8, 0, NULL), // nodes in current block |
|
53 |
_data_entry(arena(), 8, 0, NULL), // nodes with all inputs from outside |
|
54 |
_mem_slice_head(arena(), 8, 0, NULL), // memory slice heads |
|
55 |
_mem_slice_tail(arena(), 8, 0, NULL), // memory slice tails |
|
56 |
_node_info(arena(), 8, 0, SWNodeInfo::initial), // info needed per node |
|
30593 | 57 |
_clone_map(phase->C->clone_map()), // map of nodes created in cloning |
1 | 58 |
_align_to_ref(NULL), // memory reference to align vectors to |
59 |
_disjoint_ptrs(arena(), 8, 0, OrderedPair::initial), // runtime disambiguated pointer pairs |
|
60 |
_dg(_arena), // dependence graph |
|
61 |
_visited(arena()), // visited node set |
|
62 |
_post_visited(arena()), // post visited node set |
|
63 |
_n_idx_list(arena(), 8), // scratch list of (node,index) pairs |
|
64 |
_stk(arena(), 8, 0, NULL), // scratch stack of nodes |
|
65 |
_nlist(arena(), 8, 0, NULL), // scratch list of nodes |
|
66 |
_lpt(NULL), // loop tree node |
|
67 |
_lp(NULL), // LoopNode |
|
68 |
_bb(NULL), // basic block |
|
30211 | 69 |
_iv(NULL), // induction var |
30588 | 70 |
_race_possible(false), // cases where SDMU is true |
71 |
_num_work_vecs(0), // amount of vector work we have |
|
30593 | 72 |
_num_reductions(0), // amount of reduction work we have |
73 |
_do_vector_loop(phase->C->do_vector_loop()), // whether to do vectorization/simd style |
|
74 |
_ii_first(-1), // first loop generation index - only if do_vector_loop() |
|
75 |
_ii_last(-1), // last loop generation index - only if do_vector_loop() |
|
76 |
_ii_order(arena(), 8, 0, 0), |
|
77 |
_vector_loop_debug(phase->C->has_method() && phase->C->method_has_option("VectorizeDebug")) |
|
1 | 78 |
{} |
79 |
||
80 |
//------------------------------transform_loop--------------------------- |
|
81 |
void SuperWord::transform_loop(IdealLoopTree* lpt) { |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
82 |
assert(UseSuperWord, "should be"); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
83 |
// Do vectors exist on this architecture? |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
84 |
if (Matcher::vector_width_in_bytes(T_BYTE) < 2) return; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
85 |
|
1 | 86 |
assert(lpt->_head->is_CountedLoop(), "must be"); |
87 |
CountedLoopNode *cl = lpt->_head->as_CountedLoop(); |
|
88 |
||
10263
fa58671dde31
7077439: Possible reference through NULL in loopPredicate.cpp:726
kvn
parents:
10255
diff
changeset
|
89 |
if (!cl->is_valid_counted_loop()) return; // skip malformed counted loop |
fa58671dde31
7077439: Possible reference through NULL in loopPredicate.cpp:726
kvn
parents:
10255
diff
changeset
|
90 |
|
1 | 91 |
if (!cl->is_main_loop() ) return; // skip normal, pre, and post loops |
92 |
||
93 |
// Check for no control flow in body (other than exit) |
|
94 |
Node *cl_exit = cl->loopexit(); |
|
95 |
if (cl_exit->in(0) != lpt->_head) return; |
|
96 |
||
352
6d8e1aa5834e
6646020: assert(in_bb(n),"must be in block") in -Xcomp mode
never
parents:
245
diff
changeset
|
97 |
// Make sure the are no extra control users of the loop backedge |
6d8e1aa5834e
6646020: assert(in_bb(n),"must be in block") in -Xcomp mode
never
parents:
245
diff
changeset
|
98 |
if (cl->back_control()->outcnt() != 1) { |
6d8e1aa5834e
6646020: assert(in_bb(n),"must be in block") in -Xcomp mode
never
parents:
245
diff
changeset
|
99 |
return; |
6d8e1aa5834e
6646020: assert(in_bb(n),"must be in block") in -Xcomp mode
never
parents:
245
diff
changeset
|
100 |
} |
6d8e1aa5834e
6646020: assert(in_bb(n),"must be in block") in -Xcomp mode
never
parents:
245
diff
changeset
|
101 |
|
1 | 102 |
// Check for pre-loop ending with CountedLoopEnd(Bool(Cmp(x,Opaque1(limit)))) |
103 |
CountedLoopEndNode* pre_end = get_pre_loop_end(cl); |
|
104 |
if (pre_end == NULL) return; |
|
105 |
Node *pre_opaq1 = pre_end->limit(); |
|
106 |
if (pre_opaq1->Opcode() != Op_Opaque1) return; |
|
107 |
||
108 |
init(); // initialize data structures |
|
109 |
||
110 |
set_lpt(lpt); |
|
111 |
set_lp(cl); |
|
112 |
||
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
113 |
// For now, define one block which is the entire loop body |
1 | 114 |
set_bb(cl); |
115 |
||
116 |
assert(_packset.length() == 0, "packset must be empty"); |
|
117 |
SLP_extract(); |
|
118 |
} |
|
119 |
||
120 |
//------------------------------SLP_extract--------------------------- |
|
121 |
// Extract the superword level parallelism |
|
122 |
// |
|
123 |
// 1) A reverse post-order of nodes in the block is constructed. By scanning |
|
124 |
// this list from first to last, all definitions are visited before their uses. |
|
125 |
// |
|
126 |
// 2) A point-to-point dependence graph is constructed between memory references. |
|
127 |
// This simplies the upcoming "independence" checker. |
|
128 |
// |
|
129 |
// 3) The maximum depth in the node graph from the beginning of the block |
|
130 |
// to each node is computed. This is used to prune the graph search |
|
131 |
// in the independence checker. |
|
132 |
// |
|
133 |
// 4) For integer types, the necessary bit width is propagated backwards |
|
134 |
// from stores to allow packed operations on byte, char, and short |
|
135 |
// integers. This reverses the promotion to type "int" that javac |
|
136 |
// did for operations like: char c1,c2,c3; c1 = c2 + c3. |
|
137 |
// |
|
138 |
// 5) One of the memory references is picked to be an aligned vector reference. |
|
139 |
// The pre-loop trip count is adjusted to align this reference in the |
|
140 |
// unrolled body. |
|
141 |
// |
|
142 |
// 6) The initial set of pack pairs is seeded with memory references. |
|
143 |
// |
|
144 |
// 7) The set of pack pairs is extended by following use->def and def->use links. |
|
145 |
// |
|
146 |
// 8) The pairs are combined into vector sized packs. |
|
147 |
// |
|
148 |
// 9) Reorder the memory slices to co-locate members of the memory packs. |
|
149 |
// |
|
150 |
// 10) Generate ideal vector nodes for the final set of packs and where necessary, |
|
151 |
// inserting scalar promotion, vector creation from multiple scalars, and |
|
152 |
// extraction of scalar values from vectors. |
|
153 |
// |
|
154 |
void SuperWord::SLP_extract() { |
|
155 |
||
30593 | 156 |
#ifndef PRODUCT |
157 |
if (_do_vector_loop && TraceSuperWord) { |
|
158 |
tty->print("SuperWord::SLP_extract\n"); |
|
159 |
tty->print("input loop\n"); |
|
160 |
_lpt->dump_head(); |
|
161 |
_lpt->dump(); |
|
162 |
for (uint i = 0; i < _lpt->_body.size(); i++) { |
|
163 |
_lpt->_body.at(i)->dump(); |
|
164 |
} |
|
165 |
} |
|
166 |
#endif |
|
1 | 167 |
// Ready the block |
30593 | 168 |
if (!construct_bb()) { |
15755
c54cbfb4b37f
8004867: VM crashing with assert "share/vm/opto/node.hpp:357 - assert(i < _max) failed: oob"
kvn
parents:
14134
diff
changeset
|
169 |
return; // Exit if no interesting nodes or complex graph. |
30593 | 170 |
} |
171 |
// build _dg, _disjoint_ptrs |
|
1 | 172 |
dependence_graph(); |
173 |
||
30593 | 174 |
// compute function depth(Node*) |
1 | 175 |
compute_max_depth(); |
176 |
||
30593 | 177 |
if (_do_vector_loop) { |
178 |
if (mark_generations() != -1) { |
|
179 |
hoist_loads_in_graph(); // this only rebuild the graph; all basic structs need rebuild explicitly |
|
180 |
||
181 |
if (!construct_bb()) { |
|
182 |
return; // Exit if no interesting nodes or complex graph. |
|
183 |
} |
|
184 |
dependence_graph(); |
|
185 |
compute_max_depth(); |
|
186 |
} |
|
187 |
||
188 |
#ifndef PRODUCT |
|
189 |
if (TraceSuperWord) { |
|
190 |
tty->print_cr("\nSuperWord::_do_vector_loop: graph after hoist_loads_in_graph"); |
|
191 |
_lpt->dump_head(); |
|
192 |
for (int j = 0; j < _block.length(); j++) { |
|
193 |
Node* n = _block.at(j); |
|
194 |
int d = depth(n); |
|
195 |
for (int i = 0; i < d; i++) tty->print("%s", " "); |
|
196 |
tty->print("%d :", d); |
|
197 |
n->dump(); |
|
198 |
} |
|
199 |
} |
|
200 |
#endif |
|
201 |
} |
|
202 |
||
1 | 203 |
compute_vector_element_type(); |
204 |
||
205 |
// Attempt vectorization |
|
206 |
||
207 |
find_adjacent_refs(); |
|
208 |
||
209 |
extend_packlist(); |
|
210 |
||
30593 | 211 |
if (_do_vector_loop) { |
212 |
if (_packset.length() == 0) { |
|
213 |
#ifndef PRODUCT |
|
214 |
if (TraceSuperWord) { |
|
215 |
tty->print_cr("\nSuperWord::_do_vector_loop DFA could not build packset, now trying to build anyway"); |
|
216 |
} |
|
217 |
#endif |
|
218 |
pack_parallel(); |
|
219 |
} |
|
220 |
} |
|
221 |
||
1 | 222 |
combine_packs(); |
223 |
||
224 |
construct_my_pack_map(); |
|
225 |
||
226 |
filter_packs(); |
|
227 |
||
228 |
schedule(); |
|
229 |
||
230 |
output(); |
|
231 |
} |
|
232 |
||
233 |
//------------------------------find_adjacent_refs--------------------------- |
|
234 |
// Find the adjacent memory references and create pack pairs for them. |
|
235 |
// This is the initial set of packs that will then be extended by |
|
236 |
// following use->def and def->use links. The align positions are |
|
237 |
// assigned relative to the reference "align_to_ref" |
|
238 |
void SuperWord::find_adjacent_refs() { |
|
239 |
// Get list of memory operations |
|
240 |
Node_List memops; |
|
241 |
for (int i = 0; i < _block.length(); i++) { |
|
242 |
Node* n = _block.at(i); |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
243 |
if (n->is_Mem() && !n->is_LoadStore() && in_bb(n) && |
202
dc13bf0e5d5d
6633953: type2aelembytes{T_ADDRESS} should be 8 bytes in 64 bit VM
kvn
parents:
1
diff
changeset
|
244 |
is_java_primitive(n->as_Mem()->memory_type())) { |
1 | 245 |
int align = memory_alignment(n->as_Mem(), 0); |
246 |
if (align != bottom_align) { |
|
247 |
memops.push(n); |
|
248 |
} |
|
249 |
} |
|
250 |
} |
|
251 |
||
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
252 |
Node_List align_to_refs; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
253 |
int best_iv_adjustment = 0; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
254 |
MemNode* best_align_to_mem_ref = NULL; |
1 | 255 |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
256 |
while (memops.size() != 0) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
257 |
// Find a memory reference to align to. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
258 |
MemNode* mem_ref = find_align_to_ref(memops); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
259 |
if (mem_ref == NULL) break; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
260 |
align_to_refs.push(mem_ref); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
261 |
int iv_adjustment = get_iv_adjustment(mem_ref); |
1 | 262 |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
263 |
if (best_align_to_mem_ref == NULL) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
264 |
// Set memory reference which is the best from all memory operations |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
265 |
// to be used for alignment. The pre-loop trip count is modified to align |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
266 |
// this reference to a vector-aligned address. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
267 |
best_align_to_mem_ref = mem_ref; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
268 |
best_iv_adjustment = iv_adjustment; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
269 |
} |
1 | 270 |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
271 |
SWPointer align_to_ref_p(mem_ref, this); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
272 |
// Set alignment relative to "align_to_ref" for all related memory operations. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
273 |
for (int i = memops.size() - 1; i >= 0; i--) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
274 |
MemNode* s = memops.at(i)->as_Mem(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
275 |
if (isomorphic(s, mem_ref)) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
276 |
SWPointer p2(s, this); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
277 |
if (p2.comparable(align_to_ref_p)) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
278 |
int align = memory_alignment(s, iv_adjustment); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
279 |
set_alignment(s, align); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
280 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
281 |
} |
1 | 282 |
} |
283 |
||
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
284 |
// Create initial pack pairs of memory operations for which |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
285 |
// alignment is set and vectors will be aligned. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
286 |
bool create_pack = true; |
30593 | 287 |
if (memory_alignment(mem_ref, best_iv_adjustment) == 0 || _do_vector_loop) { |
13108
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
288 |
if (!Matcher::misaligned_vectors_ok()) { |
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
289 |
int vw = vector_width(mem_ref); |
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
290 |
int vw_best = vector_width(best_align_to_mem_ref); |
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
291 |
if (vw > vw_best) { |
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
292 |
// Do not vectorize a memory access with more elements per vector |
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
293 |
// if unaligned memory access is not allowed because number of |
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
294 |
// iterations in pre-loop will be not enough to align it. |
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
295 |
create_pack = false; |
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
296 |
} |
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
297 |
} |
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
298 |
} else { |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
299 |
if (same_velt_type(mem_ref, best_align_to_mem_ref)) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
300 |
// Can't allow vectorization of unaligned memory accesses with the |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
301 |
// same type since it could be overlapped accesses to the same array. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
302 |
create_pack = false; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
303 |
} else { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
304 |
// Allow independent (different type) unaligned memory operations |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
305 |
// if HW supports them. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
306 |
if (!Matcher::misaligned_vectors_ok()) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
307 |
create_pack = false; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
308 |
} else { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
309 |
// Check if packs of the same memory type but |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
310 |
// with a different alignment were created before. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
311 |
for (uint i = 0; i < align_to_refs.size(); i++) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
312 |
MemNode* mr = align_to_refs.at(i)->as_Mem(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
313 |
if (same_velt_type(mr, mem_ref) && |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
314 |
memory_alignment(mr, iv_adjustment) != 0) |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
315 |
create_pack = false; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
316 |
} |
1 | 317 |
} |
318 |
} |
|
319 |
} |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
320 |
if (create_pack) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
321 |
for (uint i = 0; i < memops.size(); i++) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
322 |
Node* s1 = memops.at(i); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
323 |
int align = alignment(s1); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
324 |
if (align == top_align) continue; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
325 |
for (uint j = 0; j < memops.size(); j++) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
326 |
Node* s2 = memops.at(j); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
327 |
if (alignment(s2) == top_align) continue; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
328 |
if (s1 != s2 && are_adjacent_refs(s1, s2)) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
329 |
if (stmts_can_pack(s1, s2, align)) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
330 |
Node_List* pair = new Node_List(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
331 |
pair->push(s1); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
332 |
pair->push(s2); |
30593 | 333 |
if (!_do_vector_loop || _clone_map.idx(s1->_idx) == _clone_map.idx(s2->_idx)) { |
334 |
_packset.append(pair); |
|
335 |
} |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
336 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
337 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
338 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
339 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
340 |
} else { // Don't create unaligned pack |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
341 |
// First, remove remaining memory ops of the same type from the list. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
342 |
for (int i = memops.size() - 1; i >= 0; i--) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
343 |
MemNode* s = memops.at(i)->as_Mem(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
344 |
if (same_velt_type(s, mem_ref)) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
345 |
memops.remove(i); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
346 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
347 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
348 |
|
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
349 |
// Second, remove already constructed packs of the same type. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
350 |
for (int i = _packset.length() - 1; i >= 0; i--) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
351 |
Node_List* p = _packset.at(i); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
352 |
MemNode* s = p->at(0)->as_Mem(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
353 |
if (same_velt_type(s, mem_ref)) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
354 |
remove_pack_at(i); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
355 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
356 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
357 |
|
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
358 |
// If needed find the best memory reference for loop alignment again. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
359 |
if (same_velt_type(mem_ref, best_align_to_mem_ref)) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
360 |
// Put memory ops from remaining packs back on memops list for |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
361 |
// the best alignment search. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
362 |
uint orig_msize = memops.size(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
363 |
for (int i = 0; i < _packset.length(); i++) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
364 |
Node_List* p = _packset.at(i); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
365 |
MemNode* s = p->at(0)->as_Mem(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
366 |
assert(!same_velt_type(s, mem_ref), "sanity"); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
367 |
memops.push(s); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
368 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
369 |
MemNode* best_align_to_mem_ref = find_align_to_ref(memops); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
370 |
if (best_align_to_mem_ref == NULL) break; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
371 |
best_iv_adjustment = get_iv_adjustment(best_align_to_mem_ref); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
372 |
// Restore list. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
373 |
while (memops.size() > orig_msize) |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
374 |
(void)memops.pop(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
375 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
376 |
} // unaligned memory accesses |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
377 |
|
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
378 |
// Remove used mem nodes. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
379 |
for (int i = memops.size() - 1; i >= 0; i--) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
380 |
MemNode* m = memops.at(i)->as_Mem(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
381 |
if (alignment(m) != top_align) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
382 |
memops.remove(i); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
383 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
384 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
385 |
|
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
386 |
} // while (memops.size() != 0 |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
387 |
set_align_to_ref(best_align_to_mem_ref); |
1 | 388 |
|
389 |
#ifndef PRODUCT |
|
390 |
if (TraceSuperWord) { |
|
391 |
tty->print_cr("\nAfter find_adjacent_refs"); |
|
392 |
print_packset(); |
|
393 |
} |
|
394 |
#endif |
|
395 |
} |
|
396 |
||
397 |
//------------------------------find_align_to_ref--------------------------- |
|
398 |
// Find a memory reference to align the loop induction variable to. |
|
399 |
// Looks first at stores then at loads, looking for a memory reference |
|
400 |
// with the largest number of references similar to it. |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
401 |
MemNode* SuperWord::find_align_to_ref(Node_List &memops) { |
1 | 402 |
GrowableArray<int> cmp_ct(arena(), memops.size(), memops.size(), 0); |
403 |
||
404 |
// Count number of comparable memory ops |
|
405 |
for (uint i = 0; i < memops.size(); i++) { |
|
406 |
MemNode* s1 = memops.at(i)->as_Mem(); |
|
407 |
SWPointer p1(s1, this); |
|
408 |
// Discard if pre loop can't align this reference |
|
409 |
if (!ref_is_alignable(p1)) { |
|
410 |
*cmp_ct.adr_at(i) = 0; |
|
411 |
continue; |
|
412 |
} |
|
413 |
for (uint j = i+1; j < memops.size(); j++) { |
|
414 |
MemNode* s2 = memops.at(j)->as_Mem(); |
|
415 |
if (isomorphic(s1, s2)) { |
|
416 |
SWPointer p2(s2, this); |
|
417 |
if (p1.comparable(p2)) { |
|
418 |
(*cmp_ct.adr_at(i))++; |
|
419 |
(*cmp_ct.adr_at(j))++; |
|
420 |
} |
|
421 |
} |
|
422 |
} |
|
423 |
} |
|
424 |
||
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
425 |
// Find Store (or Load) with the greatest number of "comparable" references, |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
426 |
// biggest vector size, smallest data size and smallest iv offset. |
1 | 427 |
int max_ct = 0; |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
428 |
int max_vw = 0; |
1 | 429 |
int max_idx = -1; |
430 |
int min_size = max_jint; |
|
431 |
int min_iv_offset = max_jint; |
|
432 |
for (uint j = 0; j < memops.size(); j++) { |
|
433 |
MemNode* s = memops.at(j)->as_Mem(); |
|
434 |
if (s->is_Store()) { |
|
13108
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
435 |
int vw = vector_width_in_bytes(s); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
436 |
assert(vw > 1, "sanity"); |
1 | 437 |
SWPointer p(s, this); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
438 |
if (cmp_ct.at(j) > max_ct || |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
439 |
cmp_ct.at(j) == max_ct && |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
440 |
(vw > max_vw || |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
441 |
vw == max_vw && |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
442 |
(data_size(s) < min_size || |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
443 |
data_size(s) == min_size && |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
444 |
(p.offset_in_bytes() < min_iv_offset)))) { |
1 | 445 |
max_ct = cmp_ct.at(j); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
446 |
max_vw = vw; |
1 | 447 |
max_idx = j; |
448 |
min_size = data_size(s); |
|
449 |
min_iv_offset = p.offset_in_bytes(); |
|
450 |
} |
|
451 |
} |
|
452 |
} |
|
453 |
// If no stores, look at loads |
|
454 |
if (max_ct == 0) { |
|
455 |
for (uint j = 0; j < memops.size(); j++) { |
|
456 |
MemNode* s = memops.at(j)->as_Mem(); |
|
457 |
if (s->is_Load()) { |
|
13108
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
458 |
int vw = vector_width_in_bytes(s); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
459 |
assert(vw > 1, "sanity"); |
1 | 460 |
SWPointer p(s, this); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
461 |
if (cmp_ct.at(j) > max_ct || |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
462 |
cmp_ct.at(j) == max_ct && |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
463 |
(vw > max_vw || |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
464 |
vw == max_vw && |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
465 |
(data_size(s) < min_size || |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
466 |
data_size(s) == min_size && |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
467 |
(p.offset_in_bytes() < min_iv_offset)))) { |
1 | 468 |
max_ct = cmp_ct.at(j); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
469 |
max_vw = vw; |
1 | 470 |
max_idx = j; |
471 |
min_size = data_size(s); |
|
472 |
min_iv_offset = p.offset_in_bytes(); |
|
473 |
} |
|
474 |
} |
|
475 |
} |
|
476 |
} |
|
477 |
||
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
478 |
#ifdef ASSERT |
1 | 479 |
if (TraceSuperWord && Verbose) { |
30593 | 480 |
tty->print_cr("\nVector memops after find_align_to_ref"); |
1 | 481 |
for (uint i = 0; i < memops.size(); i++) { |
482 |
MemNode* s = memops.at(i)->as_Mem(); |
|
483 |
s->dump(); |
|
484 |
} |
|
485 |
} |
|
486 |
#endif |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
487 |
|
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
488 |
if (max_ct > 0) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
489 |
#ifdef ASSERT |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
490 |
if (TraceSuperWord) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
491 |
tty->print("\nVector align to node: "); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
492 |
memops.at(max_idx)->as_Mem()->dump(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
493 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
494 |
#endif |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
495 |
return memops.at(max_idx)->as_Mem(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
496 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
497 |
return NULL; |
1 | 498 |
} |
499 |
||
500 |
//------------------------------ref_is_alignable--------------------------- |
|
501 |
// Can the preloop align the reference to position zero in the vector? |
|
502 |
bool SuperWord::ref_is_alignable(SWPointer& p) { |
|
503 |
if (!p.has_iv()) { |
|
504 |
return true; // no induction variable |
|
505 |
} |
|
506 |
CountedLoopEndNode* pre_end = get_pre_loop_end(lp()->as_CountedLoop()); |
|
22919
4d686766ac23
8010500: [parfait] Possible null pointer dereference at hotspot/src/share/vm/opto/loopnode.hpp
adlertz
parents:
22234
diff
changeset
|
507 |
assert(pre_end != NULL, "we must have a correct pre-loop"); |
1 | 508 |
assert(pre_end->stride_is_con(), "pre loop stride is constant"); |
509 |
int preloop_stride = pre_end->stride_con(); |
|
510 |
||
511 |
int span = preloop_stride * p.scale_in_bytes(); |
|
30215
24fb50d99dc3
8076523: assert(((ABS(iv_adjustment_in_bytes) % elt_size) == 0)) fails in superword.cpp
kvn
parents:
30211
diff
changeset
|
512 |
int mem_size = p.memory_size(); |
24fb50d99dc3
8076523: assert(((ABS(iv_adjustment_in_bytes) % elt_size) == 0)) fails in superword.cpp
kvn
parents:
30211
diff
changeset
|
513 |
int offset = p.offset_in_bytes(); |
24fb50d99dc3
8076523: assert(((ABS(iv_adjustment_in_bytes) % elt_size) == 0)) fails in superword.cpp
kvn
parents:
30211
diff
changeset
|
514 |
// Stride one accesses are alignable if offset is aligned to memory operation size. |
24fb50d99dc3
8076523: assert(((ABS(iv_adjustment_in_bytes) % elt_size) == 0)) fails in superword.cpp
kvn
parents:
30211
diff
changeset
|
515 |
// Offset can be unaligned when UseUnalignedAccesses is used. |
24fb50d99dc3
8076523: assert(((ABS(iv_adjustment_in_bytes) % elt_size) == 0)) fails in superword.cpp
kvn
parents:
30211
diff
changeset
|
516 |
if (ABS(span) == mem_size && (ABS(offset) % mem_size) == 0) { |
1 | 517 |
return true; |
30215
24fb50d99dc3
8076523: assert(((ABS(iv_adjustment_in_bytes) % elt_size) == 0)) fails in superword.cpp
kvn
parents:
30211
diff
changeset
|
518 |
} |
1 | 519 |
// If initial offset from start of object is computable, |
520 |
// compute alignment within the vector. |
|
13108
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
521 |
int vw = vector_width_in_bytes(p.mem()); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
522 |
assert(vw > 1, "sanity"); |
1 | 523 |
if (vw % span == 0) { |
524 |
Node* init_nd = pre_end->init_trip(); |
|
525 |
if (init_nd->is_Con() && p.invar() == NULL) { |
|
526 |
int init = init_nd->bottom_type()->is_int()->get_con(); |
|
527 |
||
30215
24fb50d99dc3
8076523: assert(((ABS(iv_adjustment_in_bytes) % elt_size) == 0)) fails in superword.cpp
kvn
parents:
30211
diff
changeset
|
528 |
int init_offset = init * p.scale_in_bytes() + offset; |
1 | 529 |
assert(init_offset >= 0, "positive offset from object start"); |
530 |
||
531 |
if (span > 0) { |
|
532 |
return (vw - (init_offset % vw)) % span == 0; |
|
533 |
} else { |
|
534 |
assert(span < 0, "nonzero stride * scale"); |
|
535 |
return (init_offset % vw) % -span == 0; |
|
536 |
} |
|
537 |
} |
|
538 |
} |
|
539 |
return false; |
|
540 |
} |
|
541 |
||
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
542 |
//---------------------------get_iv_adjustment--------------------------- |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
543 |
// Calculate loop's iv adjustment for this memory ops. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
544 |
int SuperWord::get_iv_adjustment(MemNode* mem_ref) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
545 |
SWPointer align_to_ref_p(mem_ref, this); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
546 |
int offset = align_to_ref_p.offset_in_bytes(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
547 |
int scale = align_to_ref_p.scale_in_bytes(); |
13108
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
548 |
int vw = vector_width_in_bytes(mem_ref); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
549 |
assert(vw > 1, "sanity"); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
550 |
int stride_sign = (scale * iv_stride()) > 0 ? 1 : -1; |
13885 | 551 |
// At least one iteration is executed in pre-loop by default. As result |
552 |
// several iterations are needed to align memory operations in main-loop even |
|
553 |
// if offset is 0. |
|
554 |
int iv_adjustment_in_bytes = (stride_sign * vw - (offset % vw)); |
|
555 |
int elt_size = align_to_ref_p.memory_size(); |
|
556 |
assert(((ABS(iv_adjustment_in_bytes) % elt_size) == 0), |
|
557 |
err_msg_res("(%d) should be divisible by (%d)", iv_adjustment_in_bytes, elt_size)); |
|
558 |
int iv_adjustment = iv_adjustment_in_bytes/elt_size; |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
559 |
|
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
560 |
#ifndef PRODUCT |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
561 |
if (TraceSuperWord) |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
562 |
tty->print_cr("\noffset = %d iv_adjust = %d elt_size = %d scale = %d iv_stride = %d vect_size %d", |
13885 | 563 |
offset, iv_adjustment, elt_size, scale, iv_stride(), vw); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
564 |
#endif |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
565 |
return iv_adjustment; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
566 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
567 |
|
1 | 568 |
//---------------------------dependence_graph--------------------------- |
569 |
// Construct dependency graph. |
|
570 |
// Add dependence edges to load/store nodes for memory dependence |
|
571 |
// A.out()->DependNode.in(1) and DependNode.out()->B.prec(x) |
|
572 |
void SuperWord::dependence_graph() { |
|
573 |
// First, assign a dependence node to each memory node |
|
574 |
for (int i = 0; i < _block.length(); i++ ) { |
|
575 |
Node *n = _block.at(i); |
|
576 |
if (n->is_Mem() || n->is_Phi() && n->bottom_type() == Type::MEMORY) { |
|
577 |
_dg.make_node(n); |
|
578 |
} |
|
579 |
} |
|
580 |
||
581 |
// For each memory slice, create the dependences |
|
582 |
for (int i = 0; i < _mem_slice_head.length(); i++) { |
|
583 |
Node* n = _mem_slice_head.at(i); |
|
584 |
Node* n_tail = _mem_slice_tail.at(i); |
|
585 |
||
586 |
// Get slice in predecessor order (last is first) |
|
587 |
mem_slice_preds(n_tail, n, _nlist); |
|
588 |
||
30593 | 589 |
#ifndef PRODUCT |
590 |
if(TraceSuperWord && Verbose) { |
|
591 |
tty->print_cr("SuperWord::dependence_graph: built a new mem slice"); |
|
592 |
for (int j = _nlist.length() - 1; j >= 0 ; j--) { |
|
593 |
_nlist.at(j)->dump(); |
|
594 |
} |
|
595 |
} |
|
596 |
#endif |
|
1 | 597 |
// Make the slice dependent on the root |
598 |
DepMem* slice = _dg.dep(n); |
|
599 |
_dg.make_edge(_dg.root(), slice); |
|
600 |
||
601 |
// Create a sink for the slice |
|
602 |
DepMem* slice_sink = _dg.make_node(NULL); |
|
603 |
_dg.make_edge(slice_sink, _dg.tail()); |
|
604 |
||
605 |
// Now visit each pair of memory ops, creating the edges |
|
606 |
for (int j = _nlist.length() - 1; j >= 0 ; j--) { |
|
607 |
Node* s1 = _nlist.at(j); |
|
608 |
||
609 |
// If no dependency yet, use slice |
|
610 |
if (_dg.dep(s1)->in_cnt() == 0) { |
|
611 |
_dg.make_edge(slice, s1); |
|
612 |
} |
|
613 |
SWPointer p1(s1->as_Mem(), this); |
|
614 |
bool sink_dependent = true; |
|
615 |
for (int k = j - 1; k >= 0; k--) { |
|
616 |
Node* s2 = _nlist.at(k); |
|
617 |
if (s1->is_Load() && s2->is_Load()) |
|
618 |
continue; |
|
619 |
SWPointer p2(s2->as_Mem(), this); |
|
620 |
||
621 |
int cmp = p1.cmp(p2); |
|
622 |
if (SuperWordRTDepCheck && |
|
623 |
p1.base() != p2.base() && p1.valid() && p2.valid()) { |
|
624 |
// Create a runtime check to disambiguate |
|
625 |
OrderedPair pp(p1.base(), p2.base()); |
|
626 |
_disjoint_ptrs.append_if_missing(pp); |
|
627 |
} else if (!SWPointer::not_equal(cmp)) { |
|
628 |
// Possibly same address |
|
629 |
_dg.make_edge(s1, s2); |
|
630 |
sink_dependent = false; |
|
631 |
} |
|
632 |
} |
|
633 |
if (sink_dependent) { |
|
634 |
_dg.make_edge(s1, slice_sink); |
|
635 |
} |
|
636 |
} |
|
637 |
#ifndef PRODUCT |
|
638 |
if (TraceSuperWord) { |
|
639 |
tty->print_cr("\nDependence graph for slice: %d", n->_idx); |
|
640 |
for (int q = 0; q < _nlist.length(); q++) { |
|
641 |
_dg.print(_nlist.at(q)); |
|
642 |
} |
|
643 |
tty->cr(); |
|
644 |
} |
|
645 |
#endif |
|
646 |
_nlist.clear(); |
|
647 |
} |
|
648 |
||
649 |
#ifndef PRODUCT |
|
650 |
if (TraceSuperWord) { |
|
651 |
tty->print_cr("\ndisjoint_ptrs: %s", _disjoint_ptrs.length() > 0 ? "" : "NONE"); |
|
652 |
for (int r = 0; r < _disjoint_ptrs.length(); r++) { |
|
653 |
_disjoint_ptrs.at(r).print(); |
|
654 |
tty->cr(); |
|
655 |
} |
|
656 |
tty->cr(); |
|
657 |
} |
|
658 |
#endif |
|
659 |
} |
|
660 |
||
661 |
//---------------------------mem_slice_preds--------------------------- |
|
662 |
// Return a memory slice (node list) in predecessor order starting at "start" |
|
663 |
void SuperWord::mem_slice_preds(Node* start, Node* stop, GrowableArray<Node*> &preds) { |
|
664 |
assert(preds.length() == 0, "start empty"); |
|
665 |
Node* n = start; |
|
666 |
Node* prev = NULL; |
|
667 |
while (true) { |
|
668 |
assert(in_bb(n), "must be in block"); |
|
669 |
for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { |
|
670 |
Node* out = n->fast_out(i); |
|
671 |
if (out->is_Load()) { |
|
672 |
if (in_bb(out)) { |
|
673 |
preds.push(out); |
|
674 |
} |
|
675 |
} else { |
|
676 |
// FIXME |
|
677 |
if (out->is_MergeMem() && !in_bb(out)) { |
|
678 |
// Either unrolling is causing a memory edge not to disappear, |
|
679 |
// or need to run igvn.optimize() again before SLP |
|
680 |
} else if (out->is_Phi() && out->bottom_type() == Type::MEMORY && !in_bb(out)) { |
|
681 |
// Ditto. Not sure what else to check further. |
|
2334 | 682 |
} else if (out->Opcode() == Op_StoreCM && out->in(MemNode::OopStore) == n) { |
1 | 683 |
// StoreCM has an input edge used as a precedence edge. |
684 |
// Maybe an issue when oop stores are vectorized. |
|
685 |
} else { |
|
686 |
assert(out == prev || prev == NULL, "no branches off of store slice"); |
|
687 |
} |
|
688 |
} |
|
689 |
} |
|
690 |
if (n == stop) break; |
|
691 |
preds.push(n); |
|
692 |
prev = n; |
|
15755
c54cbfb4b37f
8004867: VM crashing with assert "share/vm/opto/node.hpp:357 - assert(i < _max) failed: oob"
kvn
parents:
14134
diff
changeset
|
693 |
assert(n->is_Mem(), err_msg_res("unexpected node %s", n->Name())); |
1 | 694 |
n = n->in(MemNode::Memory); |
695 |
} |
|
696 |
} |
|
697 |
||
698 |
//------------------------------stmts_can_pack--------------------------- |
|
2131 | 699 |
// Can s1 and s2 be in a pack with s1 immediately preceding s2 and |
1 | 700 |
// s1 aligned at "align" |
701 |
bool SuperWord::stmts_can_pack(Node* s1, Node* s2, int align) { |
|
3906
6767b0c66883
6879921: CTW failure jdk6_18/hotspot/src/share/vm/utilities/globalDefinitions.cpp:268
cfang
parents:
3904
diff
changeset
|
702 |
|
6767b0c66883
6879921: CTW failure jdk6_18/hotspot/src/share/vm/utilities/globalDefinitions.cpp:268
cfang
parents:
3904
diff
changeset
|
703 |
// Do not use superword for non-primitives |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
704 |
BasicType bt1 = velt_basic_type(s1); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
705 |
BasicType bt2 = velt_basic_type(s2); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
706 |
if(!is_java_primitive(bt1) || !is_java_primitive(bt2)) |
3906
6767b0c66883
6879921: CTW failure jdk6_18/hotspot/src/share/vm/utilities/globalDefinitions.cpp:268
cfang
parents:
3904
diff
changeset
|
707 |
return false; |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
708 |
if (Matcher::max_vector_size(bt1) < 2) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
709 |
return false; // No vectors for this type |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
710 |
} |
3906
6767b0c66883
6879921: CTW failure jdk6_18/hotspot/src/share/vm/utilities/globalDefinitions.cpp:268
cfang
parents:
3904
diff
changeset
|
711 |
|
1 | 712 |
if (isomorphic(s1, s2)) { |
30211 | 713 |
if (independent(s1, s2) || reduction(s1, s2)) { |
1 | 714 |
if (!exists_at(s1, 0) && !exists_at(s2, 1)) { |
715 |
if (!s1->is_Mem() || are_adjacent_refs(s1, s2)) { |
|
716 |
int s1_align = alignment(s1); |
|
717 |
int s2_align = alignment(s2); |
|
718 |
if (s1_align == top_align || s1_align == align) { |
|
719 |
if (s2_align == top_align || s2_align == align + data_size(s1)) { |
|
720 |
return true; |
|
721 |
} |
|
722 |
} |
|
723 |
} |
|
724 |
} |
|
725 |
} |
|
726 |
} |
|
727 |
return false; |
|
728 |
} |
|
729 |
||
730 |
//------------------------------exists_at--------------------------- |
|
731 |
// Does s exist in a pack at position pos? |
|
732 |
bool SuperWord::exists_at(Node* s, uint pos) { |
|
733 |
for (int i = 0; i < _packset.length(); i++) { |
|
734 |
Node_List* p = _packset.at(i); |
|
735 |
if (p->at(pos) == s) { |
|
736 |
return true; |
|
737 |
} |
|
738 |
} |
|
739 |
return false; |
|
740 |
} |
|
741 |
||
742 |
//------------------------------are_adjacent_refs--------------------------- |
|
743 |
// Is s1 immediately before s2 in memory? |
|
744 |
bool SuperWord::are_adjacent_refs(Node* s1, Node* s2) { |
|
745 |
if (!s1->is_Mem() || !s2->is_Mem()) return false; |
|
746 |
if (!in_bb(s1) || !in_bb(s2)) return false; |
|
5708 | 747 |
|
748 |
// Do not use superword for non-primitives |
|
749 |
if (!is_java_primitive(s1->as_Mem()->memory_type()) || |
|
750 |
!is_java_primitive(s2->as_Mem()->memory_type())) { |
|
751 |
return false; |
|
752 |
} |
|
753 |
||
1 | 754 |
// FIXME - co_locate_pack fails on Stores in different mem-slices, so |
755 |
// only pack memops that are in the same alias set until that's fixed. |
|
756 |
if (_phase->C->get_alias_index(s1->as_Mem()->adr_type()) != |
|
757 |
_phase->C->get_alias_index(s2->as_Mem()->adr_type())) |
|
758 |
return false; |
|
759 |
SWPointer p1(s1->as_Mem(), this); |
|
760 |
SWPointer p2(s2->as_Mem(), this); |
|
761 |
if (p1.base() != p2.base() || !p1.comparable(p2)) return false; |
|
762 |
int diff = p2.offset_in_bytes() - p1.offset_in_bytes(); |
|
763 |
return diff == data_size(s1); |
|
764 |
} |
|
765 |
||
766 |
//------------------------------isomorphic--------------------------- |
|
767 |
// Are s1 and s2 similar? |
|
768 |
bool SuperWord::isomorphic(Node* s1, Node* s2) { |
|
769 |
if (s1->Opcode() != s2->Opcode()) return false; |
|
770 |
if (s1->req() != s2->req()) return false; |
|
771 |
if (s1->in(0) != s2->in(0)) return false; |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
772 |
if (!same_velt_type(s1, s2)) return false; |
1 | 773 |
return true; |
774 |
} |
|
775 |
||
776 |
//------------------------------independent--------------------------- |
|
777 |
// Is there no data path from s1 to s2 or s2 to s1? |
|
778 |
bool SuperWord::independent(Node* s1, Node* s2) { |
|
779 |
// assert(s1->Opcode() == s2->Opcode(), "check isomorphic first"); |
|
780 |
int d1 = depth(s1); |
|
781 |
int d2 = depth(s2); |
|
782 |
if (d1 == d2) return s1 != s2; |
|
783 |
Node* deep = d1 > d2 ? s1 : s2; |
|
784 |
Node* shallow = d1 > d2 ? s2 : s1; |
|
785 |
||
786 |
visited_clear(); |
|
787 |
||
788 |
return independent_path(shallow, deep); |
|
789 |
} |
|
790 |
||
30211 | 791 |
//------------------------------reduction--------------------------- |
792 |
// Is there a data path between s1 and s2 and the nodes reductions? |
|
793 |
bool SuperWord::reduction(Node* s1, Node* s2) { |
|
794 |
bool retValue = false; |
|
795 |
int d1 = depth(s1); |
|
796 |
int d2 = depth(s2); |
|
797 |
if (d1 + 1 == d2) { |
|
798 |
if (s1->is_reduction() && s2->is_reduction()) { |
|
799 |
// This is an ordered set, so s1 should define s2 |
|
800 |
for (DUIterator_Fast imax, i = s1->fast_outs(imax); i < imax; i++) { |
|
801 |
Node* t1 = s1->fast_out(i); |
|
802 |
if (t1 == s2) { |
|
803 |
// both nodes are reductions and connected |
|
804 |
retValue = true; |
|
805 |
} |
|
806 |
} |
|
807 |
} |
|
808 |
} |
|
809 |
||
810 |
return retValue; |
|
811 |
} |
|
812 |
||
1 | 813 |
//------------------------------independent_path------------------------------ |
814 |
// Helper for independent |
|
815 |
bool SuperWord::independent_path(Node* shallow, Node* deep, uint dp) { |
|
816 |
if (dp >= 1000) return false; // stop deep recursion |
|
817 |
visited_set(deep); |
|
818 |
int shal_depth = depth(shallow); |
|
819 |
assert(shal_depth <= depth(deep), "must be"); |
|
820 |
for (DepPreds preds(deep, _dg); !preds.done(); preds.next()) { |
|
821 |
Node* pred = preds.current(); |
|
822 |
if (in_bb(pred) && !visited_test(pred)) { |
|
823 |
if (shallow == pred) { |
|
824 |
return false; |
|
825 |
} |
|
826 |
if (shal_depth < depth(pred) && !independent_path(shallow, pred, dp+1)) { |
|
827 |
return false; |
|
828 |
} |
|
829 |
} |
|
830 |
} |
|
831 |
return true; |
|
832 |
} |
|
833 |
||
834 |
//------------------------------set_alignment--------------------------- |
|
835 |
void SuperWord::set_alignment(Node* s1, Node* s2, int align) { |
|
836 |
set_alignment(s1, align); |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
837 |
if (align == top_align || align == bottom_align) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
838 |
set_alignment(s2, align); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
839 |
} else { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
840 |
set_alignment(s2, align + data_size(s1)); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
841 |
} |
1 | 842 |
} |
843 |
||
844 |
//------------------------------data_size--------------------------- |
|
845 |
int SuperWord::data_size(Node* s) { |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
846 |
int bsize = type2aelembytes(velt_basic_type(s)); |
1 | 847 |
assert(bsize != 0, "valid size"); |
848 |
return bsize; |
|
849 |
} |
|
850 |
||
851 |
//------------------------------extend_packlist--------------------------- |
|
852 |
// Extend packset by following use->def and def->use links from pack members. |
|
853 |
void SuperWord::extend_packlist() { |
|
854 |
bool changed; |
|
855 |
do { |
|
30211 | 856 |
packset_sort(_packset.length()); |
1 | 857 |
changed = false; |
858 |
for (int i = 0; i < _packset.length(); i++) { |
|
859 |
Node_List* p = _packset.at(i); |
|
860 |
changed |= follow_use_defs(p); |
|
861 |
changed |= follow_def_uses(p); |
|
862 |
} |
|
863 |
} while (changed); |
|
864 |
||
30211 | 865 |
if (_race_possible) { |
866 |
for (int i = 0; i < _packset.length(); i++) { |
|
867 |
Node_List* p = _packset.at(i); |
|
868 |
order_def_uses(p); |
|
869 |
} |
|
870 |
} |
|
871 |
||
1 | 872 |
#ifndef PRODUCT |
873 |
if (TraceSuperWord) { |
|
874 |
tty->print_cr("\nAfter extend_packlist"); |
|
875 |
print_packset(); |
|
876 |
} |
|
877 |
#endif |
|
878 |
} |
|
879 |
||
880 |
//------------------------------follow_use_defs--------------------------- |
|
881 |
// Extend the packset by visiting operand definitions of nodes in pack p |
|
882 |
bool SuperWord::follow_use_defs(Node_List* p) { |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
883 |
assert(p->size() == 2, "just checking"); |
1 | 884 |
Node* s1 = p->at(0); |
885 |
Node* s2 = p->at(1); |
|
886 |
assert(s1->req() == s2->req(), "just checking"); |
|
887 |
assert(alignment(s1) + data_size(s1) == alignment(s2), "just checking"); |
|
888 |
||
889 |
if (s1->is_Load()) return false; |
|
890 |
||
891 |
int align = alignment(s1); |
|
892 |
bool changed = false; |
|
893 |
int start = s1->is_Store() ? MemNode::ValueIn : 1; |
|
894 |
int end = s1->is_Store() ? MemNode::ValueIn+1 : s1->req(); |
|
895 |
for (int j = start; j < end; j++) { |
|
896 |
Node* t1 = s1->in(j); |
|
897 |
Node* t2 = s2->in(j); |
|
898 |
if (!in_bb(t1) || !in_bb(t2)) |
|
899 |
continue; |
|
900 |
if (stmts_can_pack(t1, t2, align)) { |
|
901 |
if (est_savings(t1, t2) >= 0) { |
|
902 |
Node_List* pair = new Node_List(); |
|
903 |
pair->push(t1); |
|
904 |
pair->push(t2); |
|
905 |
_packset.append(pair); |
|
906 |
set_alignment(t1, t2, align); |
|
907 |
changed = true; |
|
908 |
} |
|
909 |
} |
|
910 |
} |
|
911 |
return changed; |
|
912 |
} |
|
913 |
||
914 |
//------------------------------follow_def_uses--------------------------- |
|
915 |
// Extend the packset by visiting uses of nodes in pack p |
|
916 |
bool SuperWord::follow_def_uses(Node_List* p) { |
|
917 |
bool changed = false; |
|
918 |
Node* s1 = p->at(0); |
|
919 |
Node* s2 = p->at(1); |
|
920 |
assert(p->size() == 2, "just checking"); |
|
921 |
assert(s1->req() == s2->req(), "just checking"); |
|
922 |
assert(alignment(s1) + data_size(s1) == alignment(s2), "just checking"); |
|
923 |
||
924 |
if (s1->is_Store()) return false; |
|
925 |
||
926 |
int align = alignment(s1); |
|
927 |
int savings = -1; |
|
30211 | 928 |
int num_s1_uses = 0; |
1 | 929 |
Node* u1 = NULL; |
930 |
Node* u2 = NULL; |
|
931 |
for (DUIterator_Fast imax, i = s1->fast_outs(imax); i < imax; i++) { |
|
932 |
Node* t1 = s1->fast_out(i); |
|
30211 | 933 |
num_s1_uses++; |
1 | 934 |
if (!in_bb(t1)) continue; |
935 |
for (DUIterator_Fast jmax, j = s2->fast_outs(jmax); j < jmax; j++) { |
|
936 |
Node* t2 = s2->fast_out(j); |
|
937 |
if (!in_bb(t2)) continue; |
|
938 |
if (!opnd_positions_match(s1, t1, s2, t2)) |
|
939 |
continue; |
|
940 |
if (stmts_can_pack(t1, t2, align)) { |
|
941 |
int my_savings = est_savings(t1, t2); |
|
942 |
if (my_savings > savings) { |
|
943 |
savings = my_savings; |
|
944 |
u1 = t1; |
|
945 |
u2 = t2; |
|
946 |
} |
|
947 |
} |
|
948 |
} |
|
949 |
} |
|
30211 | 950 |
if (num_s1_uses > 1) { |
951 |
_race_possible = true; |
|
952 |
} |
|
1 | 953 |
if (savings >= 0) { |
954 |
Node_List* pair = new Node_List(); |
|
955 |
pair->push(u1); |
|
956 |
pair->push(u2); |
|
957 |
_packset.append(pair); |
|
958 |
set_alignment(u1, u2, align); |
|
959 |
changed = true; |
|
960 |
} |
|
961 |
return changed; |
|
962 |
} |
|
963 |
||
30211 | 964 |
//------------------------------order_def_uses--------------------------- |
965 |
// For extended packsets, ordinally arrange uses packset by major component |
|
966 |
void SuperWord::order_def_uses(Node_List* p) { |
|
967 |
Node* s1 = p->at(0); |
|
968 |
||
969 |
if (s1->is_Store()) return; |
|
970 |
||
971 |
// reductions are always managed beforehand |
|
972 |
if (s1->is_reduction()) return; |
|
973 |
||
974 |
for (DUIterator_Fast imax, i = s1->fast_outs(imax); i < imax; i++) { |
|
975 |
Node* t1 = s1->fast_out(i); |
|
976 |
||
977 |
// Only allow operand swap on commuting operations |
|
978 |
if (!t1->is_Add() && !t1->is_Mul()) { |
|
979 |
break; |
|
980 |
} |
|
981 |
||
982 |
// Now find t1's packset |
|
983 |
Node_List* p2 = NULL; |
|
984 |
for (int j = 0; j < _packset.length(); j++) { |
|
985 |
p2 = _packset.at(j); |
|
986 |
Node* first = p2->at(0); |
|
987 |
if (t1 == first) { |
|
988 |
break; |
|
989 |
} |
|
990 |
p2 = NULL; |
|
991 |
} |
|
992 |
// Arrange all sub components by the major component |
|
993 |
if (p2 != NULL) { |
|
994 |
for (uint j = 1; j < p->size(); j++) { |
|
995 |
Node* d1 = p->at(j); |
|
996 |
Node* u1 = p2->at(j); |
|
997 |
opnd_positions_match(s1, t1, d1, u1); |
|
998 |
} |
|
999 |
} |
|
1000 |
} |
|
1001 |
} |
|
1002 |
||
1 | 1003 |
//---------------------------opnd_positions_match------------------------- |
1004 |
// Is the use of d1 in u1 at the same operand position as d2 in u2? |
|
1005 |
bool SuperWord::opnd_positions_match(Node* d1, Node* u1, Node* d2, Node* u2) { |
|
30211 | 1006 |
// check reductions to see if they are marshalled to represent the reduction |
1007 |
// operator in a specified opnd |
|
1008 |
if (u1->is_reduction() && u2->is_reduction()) { |
|
1009 |
// ensure reductions have phis and reduction definitions feeding the 1st operand |
|
1010 |
Node* first = u1->in(2); |
|
1011 |
if (first->is_Phi() || first->is_reduction()) { |
|
1012 |
u1->swap_edges(1, 2); |
|
1013 |
} |
|
1014 |
// ensure reductions have phis and reduction definitions feeding the 1st operand |
|
1015 |
first = u2->in(2); |
|
1016 |
if (first->is_Phi() || first->is_reduction()) { |
|
1017 |
u2->swap_edges(1, 2); |
|
1018 |
} |
|
1019 |
return true; |
|
1020 |
} |
|
1021 |
||
1 | 1022 |
uint ct = u1->req(); |
1023 |
if (ct != u2->req()) return false; |
|
1024 |
uint i1 = 0; |
|
1025 |
uint i2 = 0; |
|
1026 |
do { |
|
1027 |
for (i1++; i1 < ct; i1++) if (u1->in(i1) == d1) break; |
|
1028 |
for (i2++; i2 < ct; i2++) if (u2->in(i2) == d2) break; |
|
1029 |
if (i1 != i2) { |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1030 |
if ((i1 == (3-i2)) && (u2->is_Add() || u2->is_Mul())) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1031 |
// Further analysis relies on operands position matching. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1032 |
u2->swap_edges(i1, i2); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1033 |
} else { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1034 |
return false; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1035 |
} |
1 | 1036 |
} |
1037 |
} while (i1 < ct); |
|
1038 |
return true; |
|
1039 |
} |
|
1040 |
||
1041 |
//------------------------------est_savings--------------------------- |
|
1042 |
// Estimate the savings from executing s1 and s2 as a pack |
|
1043 |
int SuperWord::est_savings(Node* s1, Node* s2) { |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1044 |
int save_in = 2 - 1; // 2 operations per instruction in packed form |
1 | 1045 |
|
1046 |
// inputs |
|
1047 |
for (uint i = 1; i < s1->req(); i++) { |
|
1048 |
Node* x1 = s1->in(i); |
|
1049 |
Node* x2 = s2->in(i); |
|
1050 |
if (x1 != x2) { |
|
1051 |
if (are_adjacent_refs(x1, x2)) { |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1052 |
save_in += adjacent_profit(x1, x2); |
1 | 1053 |
} else if (!in_packset(x1, x2)) { |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1054 |
save_in -= pack_cost(2); |
1 | 1055 |
} else { |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1056 |
save_in += unpack_cost(2); |
1 | 1057 |
} |
1058 |
} |
|
1059 |
} |
|
1060 |
||
1061 |
// uses of result |
|
1062 |
uint ct = 0; |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1063 |
int save_use = 0; |
1 | 1064 |
for (DUIterator_Fast imax, i = s1->fast_outs(imax); i < imax; i++) { |
1065 |
Node* s1_use = s1->fast_out(i); |
|
1066 |
for (int j = 0; j < _packset.length(); j++) { |
|
1067 |
Node_List* p = _packset.at(j); |
|
1068 |
if (p->at(0) == s1_use) { |
|
1069 |
for (DUIterator_Fast kmax, k = s2->fast_outs(kmax); k < kmax; k++) { |
|
1070 |
Node* s2_use = s2->fast_out(k); |
|
1071 |
if (p->at(p->size()-1) == s2_use) { |
|
1072 |
ct++; |
|
1073 |
if (are_adjacent_refs(s1_use, s2_use)) { |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1074 |
save_use += adjacent_profit(s1_use, s2_use); |
1 | 1075 |
} |
1076 |
} |
|
1077 |
} |
|
1078 |
} |
|
1079 |
} |
|
1080 |
} |
|
1081 |
||
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1082 |
if (ct < s1->outcnt()) save_use += unpack_cost(1); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1083 |
if (ct < s2->outcnt()) save_use += unpack_cost(1); |
1 | 1084 |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1085 |
return MAX2(save_in, save_use); |
1 | 1086 |
} |
1087 |
||
1088 |
//------------------------------costs--------------------------- |
|
1089 |
int SuperWord::adjacent_profit(Node* s1, Node* s2) { return 2; } |
|
1090 |
int SuperWord::pack_cost(int ct) { return ct; } |
|
1091 |
int SuperWord::unpack_cost(int ct) { return ct; } |
|
1092 |
||
1093 |
//------------------------------combine_packs--------------------------- |
|
1094 |
// Combine packs A and B with A.last == B.first into A.first..,A.last,B.second,..B.last |
|
1095 |
void SuperWord::combine_packs() { |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1096 |
bool changed = true; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1097 |
// Combine packs regardless max vector size. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1098 |
while (changed) { |
1 | 1099 |
changed = false; |
1100 |
for (int i = 0; i < _packset.length(); i++) { |
|
1101 |
Node_List* p1 = _packset.at(i); |
|
1102 |
if (p1 == NULL) continue; |
|
30211 | 1103 |
// Because of sorting we can start at i + 1 |
1104 |
for (int j = i + 1; j < _packset.length(); j++) { |
|
1 | 1105 |
Node_List* p2 = _packset.at(j); |
1106 |
if (p2 == NULL) continue; |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1107 |
if (i == j) continue; |
1 | 1108 |
if (p1->at(p1->size()-1) == p2->at(0)) { |
1109 |
for (uint k = 1; k < p2->size(); k++) { |
|
1110 |
p1->push(p2->at(k)); |
|
1111 |
} |
|
1112 |
_packset.at_put(j, NULL); |
|
1113 |
changed = true; |
|
1114 |
} |
|
1115 |
} |
|
1116 |
} |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1117 |
} |
1 | 1118 |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1119 |
// Split packs which have size greater then max vector size. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1120 |
for (int i = 0; i < _packset.length(); i++) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1121 |
Node_List* p1 = _packset.at(i); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1122 |
if (p1 != NULL) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1123 |
BasicType bt = velt_basic_type(p1->at(0)); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1124 |
uint max_vlen = Matcher::max_vector_size(bt); // Max elements in vector |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1125 |
assert(is_power_of_2(max_vlen), "sanity"); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1126 |
uint psize = p1->size(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1127 |
if (!is_power_of_2(psize)) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1128 |
// Skip pack which can't be vector. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1129 |
// case1: for(...) { a[i] = i; } elements values are different (i+x) |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1130 |
// case2: for(...) { a[i] = b[i+1]; } can't align both, load and store |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1131 |
_packset.at_put(i, NULL); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1132 |
continue; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1133 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1134 |
if (psize > max_vlen) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1135 |
Node_List* pack = new Node_List(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1136 |
for (uint j = 0; j < psize; j++) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1137 |
pack->push(p1->at(j)); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1138 |
if (pack->size() >= max_vlen) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1139 |
assert(is_power_of_2(pack->size()), "sanity"); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1140 |
_packset.append(pack); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1141 |
pack = new Node_List(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1142 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1143 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1144 |
_packset.at_put(i, NULL); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1145 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1146 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1147 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1148 |
|
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1149 |
// Compress list. |
1 | 1150 |
for (int i = _packset.length() - 1; i >= 0; i--) { |
1151 |
Node_List* p1 = _packset.at(i); |
|
1152 |
if (p1 == NULL) { |
|
1153 |
_packset.remove_at(i); |
|
1154 |
} |
|
1155 |
} |
|
1156 |
||
1157 |
#ifndef PRODUCT |
|
1158 |
if (TraceSuperWord) { |
|
1159 |
tty->print_cr("\nAfter combine_packs"); |
|
1160 |
print_packset(); |
|
1161 |
} |
|
1162 |
#endif |
|
1163 |
} |
|
1164 |
||
1165 |
//-----------------------------construct_my_pack_map-------------------------- |
|
1166 |
// Construct the map from nodes to packs. Only valid after the |
|
1167 |
// point where a node is only in one pack (after combine_packs). |
|
1168 |
void SuperWord::construct_my_pack_map() { |
|
1169 |
Node_List* rslt = NULL; |
|
1170 |
for (int i = 0; i < _packset.length(); i++) { |
|
1171 |
Node_List* p = _packset.at(i); |
|
1172 |
for (uint j = 0; j < p->size(); j++) { |
|
1173 |
Node* s = p->at(j); |
|
1174 |
assert(my_pack(s) == NULL, "only in one pack"); |
|
1175 |
set_my_pack(s, p); |
|
1176 |
} |
|
1177 |
} |
|
1178 |
} |
|
1179 |
||
1180 |
//------------------------------filter_packs--------------------------- |
|
1181 |
// Remove packs that are not implemented or not profitable. |
|
1182 |
void SuperWord::filter_packs() { |
|
1183 |
// Remove packs that are not implemented |
|
1184 |
for (int i = _packset.length() - 1; i >= 0; i--) { |
|
1185 |
Node_List* pk = _packset.at(i); |
|
1186 |
bool impl = implemented(pk); |
|
1187 |
if (!impl) { |
|
1188 |
#ifndef PRODUCT |
|
1189 |
if (TraceSuperWord && Verbose) { |
|
1190 |
tty->print_cr("Unimplemented"); |
|
1191 |
pk->at(0)->dump(); |
|
1192 |
} |
|
1193 |
#endif |
|
1194 |
remove_pack_at(i); |
|
1195 |
} |
|
30588 | 1196 |
Node *n = pk->at(0); |
1197 |
if (n->is_reduction()) { |
|
1198 |
_num_reductions++; |
|
1199 |
} else { |
|
1200 |
_num_work_vecs++; |
|
1201 |
} |
|
1 | 1202 |
} |
1203 |
||
1204 |
// Remove packs that are not profitable |
|
1205 |
bool changed; |
|
1206 |
do { |
|
1207 |
changed = false; |
|
1208 |
for (int i = _packset.length() - 1; i >= 0; i--) { |
|
1209 |
Node_List* pk = _packset.at(i); |
|
1210 |
bool prof = profitable(pk); |
|
1211 |
if (!prof) { |
|
1212 |
#ifndef PRODUCT |
|
1213 |
if (TraceSuperWord && Verbose) { |
|
1214 |
tty->print_cr("Unprofitable"); |
|
1215 |
pk->at(0)->dump(); |
|
1216 |
} |
|
1217 |
#endif |
|
1218 |
remove_pack_at(i); |
|
1219 |
changed = true; |
|
1220 |
} |
|
1221 |
} |
|
1222 |
} while (changed); |
|
1223 |
||
1224 |
#ifndef PRODUCT |
|
1225 |
if (TraceSuperWord) { |
|
1226 |
tty->print_cr("\nAfter filter_packs"); |
|
1227 |
print_packset(); |
|
1228 |
tty->cr(); |
|
1229 |
} |
|
1230 |
#endif |
|
1231 |
} |
|
1232 |
||
1233 |
//------------------------------implemented--------------------------- |
|
1234 |
// Can code be generated for pack p? |
|
1235 |
bool SuperWord::implemented(Node_List* p) { |
|
30211 | 1236 |
bool retValue = false; |
1 | 1237 |
Node* p0 = p->at(0); |
30211 | 1238 |
if (p0 != NULL) { |
1239 |
int opc = p0->Opcode(); |
|
1240 |
uint size = p->size(); |
|
1241 |
if (p0->is_reduction()) { |
|
1242 |
const Type *arith_type = p0->bottom_type(); |
|
30588 | 1243 |
// Length 2 reductions of INT/LONG do not offer performance benefits |
1244 |
if (((arith_type->basic_type() == T_INT) || (arith_type->basic_type() == T_LONG)) && (size == 2)) { |
|
1245 |
retValue = false; |
|
1246 |
} else { |
|
1247 |
retValue = ReductionNode::implemented(opc, size, arith_type->basic_type()); |
|
1248 |
} |
|
30211 | 1249 |
} else { |
1250 |
retValue = VectorNode::implemented(opc, size, velt_basic_type(p0)); |
|
1251 |
} |
|
1252 |
} |
|
1253 |
return retValue; |
|
13490
d19348851d2e
7192963: assert(_in[req-1] == this) failed: Must pass arg count to 'new'
kvn
parents:
13488
diff
changeset
|
1254 |
} |
d19348851d2e
7192963: assert(_in[req-1] == this) failed: Must pass arg count to 'new'
kvn
parents:
13488
diff
changeset
|
1255 |
|
d19348851d2e
7192963: assert(_in[req-1] == this) failed: Must pass arg count to 'new'
kvn
parents:
13488
diff
changeset
|
1256 |
//------------------------------same_inputs-------------------------- |
d19348851d2e
7192963: assert(_in[req-1] == this) failed: Must pass arg count to 'new'
kvn
parents:
13488
diff
changeset
|
1257 |
// For pack p, are all idx operands the same? |
d19348851d2e
7192963: assert(_in[req-1] == this) failed: Must pass arg count to 'new'
kvn
parents:
13488
diff
changeset
|
1258 |
static bool same_inputs(Node_List* p, int idx) { |
d19348851d2e
7192963: assert(_in[req-1] == this) failed: Must pass arg count to 'new'
kvn
parents:
13488
diff
changeset
|
1259 |
Node* p0 = p->at(0); |
d19348851d2e
7192963: assert(_in[req-1] == this) failed: Must pass arg count to 'new'
kvn
parents:
13488
diff
changeset
|
1260 |
uint vlen = p->size(); |
d19348851d2e
7192963: assert(_in[req-1] == this) failed: Must pass arg count to 'new'
kvn
parents:
13488
diff
changeset
|
1261 |
Node* p0_def = p0->in(idx); |
d19348851d2e
7192963: assert(_in[req-1] == this) failed: Must pass arg count to 'new'
kvn
parents:
13488
diff
changeset
|
1262 |
for (uint i = 1; i < vlen; i++) { |
d19348851d2e
7192963: assert(_in[req-1] == this) failed: Must pass arg count to 'new'
kvn
parents:
13488
diff
changeset
|
1263 |
Node* pi = p->at(i); |
d19348851d2e
7192963: assert(_in[req-1] == this) failed: Must pass arg count to 'new'
kvn
parents:
13488
diff
changeset
|
1264 |
Node* pi_def = pi->in(idx); |
d19348851d2e
7192963: assert(_in[req-1] == this) failed: Must pass arg count to 'new'
kvn
parents:
13488
diff
changeset
|
1265 |
if (p0_def != pi_def) |
d19348851d2e
7192963: assert(_in[req-1] == this) failed: Must pass arg count to 'new'
kvn
parents:
13488
diff
changeset
|
1266 |
return false; |
13488 | 1267 |
} |
13490
d19348851d2e
7192963: assert(_in[req-1] == this) failed: Must pass arg count to 'new'
kvn
parents:
13488
diff
changeset
|
1268 |
return true; |
1 | 1269 |
} |
1270 |
||
1271 |
//------------------------------profitable--------------------------- |
|
1272 |
// For pack p, are all operands and all uses (with in the block) vector? |
|
1273 |
bool SuperWord::profitable(Node_List* p) { |
|
1274 |
Node* p0 = p->at(0); |
|
1275 |
uint start, end; |
|
13490
d19348851d2e
7192963: assert(_in[req-1] == this) failed: Must pass arg count to 'new'
kvn
parents:
13488
diff
changeset
|
1276 |
VectorNode::vector_operands(p0, &start, &end); |
1 | 1277 |
|
13894 | 1278 |
// Return false if some inputs are not vectors or vectors with different |
1279 |
// size or alignment. |
|
1280 |
// Also, for now, return false if not scalar promotion case when inputs are |
|
1281 |
// the same. Later, implement PackNode and allow differing, non-vector inputs |
|
1282 |
// (maybe just the ones from outside the block.) |
|
1 | 1283 |
for (uint i = start; i < end; i++) { |
13894 | 1284 |
if (!is_vector_use(p0, i)) |
1285 |
return false; |
|
1 | 1286 |
} |
30211 | 1287 |
// Check if reductions are connected |
1288 |
if (p0->is_reduction()) { |
|
1289 |
Node* second_in = p0->in(2); |
|
1290 |
Node_List* second_pk = my_pack(second_in); |
|
30588 | 1291 |
if ((second_pk == NULL) || (_num_work_vecs == _num_reductions)) { |
1292 |
// Remove reduction flag if no parent pack or if not enough work |
|
1293 |
// to cover reduction expansion overhead |
|
30211 | 1294 |
p0->remove_flag(Node::Flag_is_reduction); |
1295 |
return false; |
|
1296 |
} else if (second_pk->size() != p->size()) { |
|
1297 |
return false; |
|
1298 |
} |
|
1299 |
} |
|
13490
d19348851d2e
7192963: assert(_in[req-1] == this) failed: Must pass arg count to 'new'
kvn
parents:
13488
diff
changeset
|
1300 |
if (VectorNode::is_shift(p0)) { |
13894 | 1301 |
// For now, return false if shift count is vector or not scalar promotion |
1302 |
// case (different shift counts) because it is not supported yet. |
|
1303 |
Node* cnt = p0->in(2); |
|
1304 |
Node_List* cnt_pk = my_pack(cnt); |
|
1305 |
if (cnt_pk != NULL) |
|
13490
d19348851d2e
7192963: assert(_in[req-1] == this) failed: Must pass arg count to 'new'
kvn
parents:
13488
diff
changeset
|
1306 |
return false; |
d19348851d2e
7192963: assert(_in[req-1] == this) failed: Must pass arg count to 'new'
kvn
parents:
13488
diff
changeset
|
1307 |
if (!same_inputs(p, 2)) |
d19348851d2e
7192963: assert(_in[req-1] == this) failed: Must pass arg count to 'new'
kvn
parents:
13488
diff
changeset
|
1308 |
return false; |
d19348851d2e
7192963: assert(_in[req-1] == this) failed: Must pass arg count to 'new'
kvn
parents:
13488
diff
changeset
|
1309 |
} |
1 | 1310 |
if (!p0->is_Store()) { |
1311 |
// For now, return false if not all uses are vector. |
|
1312 |
// Later, implement ExtractNode and allow non-vector uses (maybe |
|
1313 |
// just the ones outside the block.) |
|
1314 |
for (uint i = 0; i < p->size(); i++) { |
|
1315 |
Node* def = p->at(i); |
|
1316 |
for (DUIterator_Fast jmax, j = def->fast_outs(jmax); j < jmax; j++) { |
|
1317 |
Node* use = def->fast_out(j); |
|
1318 |
for (uint k = 0; k < use->req(); k++) { |
|
1319 |
Node* n = use->in(k); |
|
1320 |
if (def == n) { |
|
30211 | 1321 |
// reductions can be loop carried dependences |
1322 |
if (def->is_reduction() && use->is_Phi()) |
|
1323 |
continue; |
|
1 | 1324 |
if (!is_vector_use(use, k)) { |
1325 |
return false; |
|
1326 |
} |
|
1327 |
} |
|
1328 |
} |
|
1329 |
} |
|
1330 |
} |
|
1331 |
} |
|
1332 |
return true; |
|
1333 |
} |
|
1334 |
||
1335 |
//------------------------------schedule--------------------------- |
|
1336 |
// Adjust the memory graph for the packed operations |
|
1337 |
void SuperWord::schedule() { |
|
1338 |
||
1339 |
// Co-locate in the memory graph the members of each memory pack |
|
1340 |
for (int i = 0; i < _packset.length(); i++) { |
|
1341 |
co_locate_pack(_packset.at(i)); |
|
1342 |
} |
|
1343 |
} |
|
1344 |
||
2334 | 1345 |
//-------------------------------remove_and_insert------------------- |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1346 |
// Remove "current" from its current position in the memory graph and insert |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1347 |
// it after the appropriate insertion point (lip or uip). |
2334 | 1348 |
void SuperWord::remove_and_insert(MemNode *current, MemNode *prev, MemNode *lip, |
1349 |
Node *uip, Unique_Node_List &sched_before) { |
|
1350 |
Node* my_mem = current->in(MemNode::Memory); |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1351 |
bool sched_up = sched_before.member(current); |
2334 | 1352 |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1353 |
// remove current_store from its current position in the memmory graph |
2334 | 1354 |
for (DUIterator i = current->outs(); current->has_out(i); i++) { |
1355 |
Node* use = current->out(i); |
|
1356 |
if (use->is_Mem()) { |
|
1357 |
assert(use->in(MemNode::Memory) == current, "must be"); |
|
1358 |
if (use == prev) { // connect prev to my_mem |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1359 |
_igvn.replace_input_of(use, MemNode::Memory, my_mem); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1360 |
--i; //deleted this edge; rescan position |
2334 | 1361 |
} else if (sched_before.member(use)) { |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1362 |
if (!sched_up) { // Will be moved together with current |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1363 |
_igvn.replace_input_of(use, MemNode::Memory, uip); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1364 |
--i; //deleted this edge; rescan position |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1365 |
} |
2334 | 1366 |
} else { |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1367 |
if (sched_up) { // Will be moved together with current |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1368 |
_igvn.replace_input_of(use, MemNode::Memory, lip); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1369 |
--i; //deleted this edge; rescan position |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1370 |
} |
2334 | 1371 |
} |
1372 |
} |
|
1373 |
} |
|
1374 |
||
1375 |
Node *insert_pt = sched_up ? uip : lip; |
|
1376 |
||
1377 |
// all uses of insert_pt's memory state should use current's instead |
|
1378 |
for (DUIterator i = insert_pt->outs(); insert_pt->has_out(i); i++) { |
|
1379 |
Node* use = insert_pt->out(i); |
|
1380 |
if (use->is_Mem()) { |
|
1381 |
assert(use->in(MemNode::Memory) == insert_pt, "must be"); |
|
12958
009b6c9586d8
7173340: C2: code cleanup: use PhaseIterGVN::replace_edge(Node*, int, Node*) where applicable
kvn
parents:
12594
diff
changeset
|
1382 |
_igvn.replace_input_of(use, MemNode::Memory, current); |
2334 | 1383 |
--i; //deleted this edge; rescan position |
1384 |
} else if (!sched_up && use->is_Phi() && use->bottom_type() == Type::MEMORY) { |
|
1385 |
uint pos; //lip (lower insert point) must be the last one in the memory slice |
|
1386 |
for (pos=1; pos < use->req(); pos++) { |
|
1387 |
if (use->in(pos) == insert_pt) break; |
|
1388 |
} |
|
12958
009b6c9586d8
7173340: C2: code cleanup: use PhaseIterGVN::replace_edge(Node*, int, Node*) where applicable
kvn
parents:
12594
diff
changeset
|
1389 |
_igvn.replace_input_of(use, pos, current); |
2334 | 1390 |
--i; |
1391 |
} |
|
1392 |
} |
|
1393 |
||
1394 |
//connect current to insert_pt |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1395 |
_igvn.replace_input_of(current, MemNode::Memory, insert_pt); |
2334 | 1396 |
} |
1397 |
||
1398 |
//------------------------------co_locate_pack---------------------------------- |
|
1399 |
// To schedule a store pack, we need to move any sandwiched memory ops either before |
|
1400 |
// or after the pack, based upon dependence information: |
|
1401 |
// (1) If any store in the pack depends on the sandwiched memory op, the |
|
1402 |
// sandwiched memory op must be scheduled BEFORE the pack; |
|
1403 |
// (2) If a sandwiched memory op depends on any store in the pack, the |
|
1404 |
// sandwiched memory op must be scheduled AFTER the pack; |
|
1405 |
// (3) If a sandwiched memory op (say, memA) depends on another sandwiched |
|
1406 |
// memory op (say memB), memB must be scheduled before memA. So, if memA is |
|
1407 |
// scheduled before the pack, memB must also be scheduled before the pack; |
|
1408 |
// (4) If there is no dependence restriction for a sandwiched memory op, we simply |
|
1409 |
// schedule this store AFTER the pack |
|
1410 |
// (5) We know there is no dependence cycle, so there in no other case; |
|
1411 |
// (6) Finally, all memory ops in another single pack should be moved in the same direction. |
|
1412 |
// |
|
3799 | 1413 |
// To schedule a load pack, we use the memory state of either the first or the last load in |
1414 |
// the pack, based on the dependence constraint. |
|
1 | 1415 |
void SuperWord::co_locate_pack(Node_List* pk) { |
1416 |
if (pk->at(0)->is_Store()) { |
|
1417 |
MemNode* first = executed_first(pk)->as_Mem(); |
|
1418 |
MemNode* last = executed_last(pk)->as_Mem(); |
|
2334 | 1419 |
Unique_Node_List schedule_before_pack; |
1420 |
Unique_Node_List memops; |
|
1421 |
||
1 | 1422 |
MemNode* current = last->in(MemNode::Memory)->as_Mem(); |
2334 | 1423 |
MemNode* previous = last; |
1 | 1424 |
while (true) { |
1425 |
assert(in_bb(current), "stay in block"); |
|
2334 | 1426 |
memops.push(previous); |
1427 |
for (DUIterator i = current->outs(); current->has_out(i); i++) { |
|
1428 |
Node* use = current->out(i); |
|
1429 |
if (use->is_Mem() && use != previous) |
|
1430 |
memops.push(use); |
|
1431 |
} |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1432 |
if (current == first) break; |
2334 | 1433 |
previous = current; |
1434 |
current = current->in(MemNode::Memory)->as_Mem(); |
|
1435 |
} |
|
1436 |
||
1437 |
// determine which memory operations should be scheduled before the pack |
|
1438 |
for (uint i = 1; i < memops.size(); i++) { |
|
1439 |
Node *s1 = memops.at(i); |
|
1440 |
if (!in_pack(s1, pk) && !schedule_before_pack.member(s1)) { |
|
1441 |
for (uint j = 0; j< i; j++) { |
|
1442 |
Node *s2 = memops.at(j); |
|
1443 |
if (!independent(s1, s2)) { |
|
1444 |
if (in_pack(s2, pk) || schedule_before_pack.member(s2)) { |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1445 |
schedule_before_pack.push(s1); // s1 must be scheduled before |
2334 | 1446 |
Node_List* mem_pk = my_pack(s1); |
1447 |
if (mem_pk != NULL) { |
|
1448 |
for (uint ii = 0; ii < mem_pk->size(); ii++) { |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1449 |
Node* s = mem_pk->at(ii); // follow partner |
2334 | 1450 |
if (memops.member(s) && !schedule_before_pack.member(s)) |
1451 |
schedule_before_pack.push(s); |
|
1452 |
} |
|
1453 |
} |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1454 |
break; |
2334 | 1455 |
} |
1456 |
} |
|
1457 |
} |
|
1458 |
} |
|
1459 |
} |
|
1460 |
||
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1461 |
Node* upper_insert_pt = first->in(MemNode::Memory); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1462 |
// Following code moves loads connected to upper_insert_pt below aliased stores. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1463 |
// Collect such loads here and reconnect them back to upper_insert_pt later. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1464 |
memops.clear(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1465 |
for (DUIterator i = upper_insert_pt->outs(); upper_insert_pt->has_out(i); i++) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1466 |
Node* use = upper_insert_pt->out(i); |
24090
196e4bd91543
8041351: Crash in src/share/vm/opto/loopnode.cpp:3215 - assert(!had_error) failed: bad dominance
kvn
parents:
23528
diff
changeset
|
1467 |
if (use->is_Mem() && !use->is_Store()) { |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1468 |
memops.push(use); |
24090
196e4bd91543
8041351: Crash in src/share/vm/opto/loopnode.cpp:3215 - assert(!had_error) failed: bad dominance
kvn
parents:
23528
diff
changeset
|
1469 |
} |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1470 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1471 |
|
2334 | 1472 |
MemNode* lower_insert_pt = last; |
1473 |
previous = last; //previous store in pk |
|
1474 |
current = last->in(MemNode::Memory)->as_Mem(); |
|
1475 |
||
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1476 |
// start scheduling from "last" to "first" |
2334 | 1477 |
while (true) { |
1478 |
assert(in_bb(current), "stay in block"); |
|
1479 |
assert(in_pack(previous, pk), "previous stays in pack"); |
|
1 | 1480 |
Node* my_mem = current->in(MemNode::Memory); |
2334 | 1481 |
|
1 | 1482 |
if (in_pack(current, pk)) { |
2334 | 1483 |
// Forward users of my memory state (except "previous) to my input memory state |
1 | 1484 |
for (DUIterator i = current->outs(); current->has_out(i); i++) { |
1485 |
Node* use = current->out(i); |
|
2334 | 1486 |
if (use->is_Mem() && use != previous) { |
1 | 1487 |
assert(use->in(MemNode::Memory) == current, "must be"); |
2334 | 1488 |
if (schedule_before_pack.member(use)) { |
12958
009b6c9586d8
7173340: C2: code cleanup: use PhaseIterGVN::replace_edge(Node*, int, Node*) where applicable
kvn
parents:
12594
diff
changeset
|
1489 |
_igvn.replace_input_of(use, MemNode::Memory, upper_insert_pt); |
2334 | 1490 |
} else { |
12958
009b6c9586d8
7173340: C2: code cleanup: use PhaseIterGVN::replace_edge(Node*, int, Node*) where applicable
kvn
parents:
12594
diff
changeset
|
1491 |
_igvn.replace_input_of(use, MemNode::Memory, lower_insert_pt); |
2334 | 1492 |
} |
1 | 1493 |
--i; // deleted this edge; rescan position |
1494 |
} |
|
1495 |
} |
|
2334 | 1496 |
previous = current; |
1497 |
} else { // !in_pack(current, pk) ==> a sandwiched store |
|
1498 |
remove_and_insert(current, previous, lower_insert_pt, upper_insert_pt, schedule_before_pack); |
|
1 | 1499 |
} |
2334 | 1500 |
|
1 | 1501 |
if (current == first) break; |
1502 |
current = my_mem->as_Mem(); |
|
2334 | 1503 |
} // end while |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1504 |
|
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1505 |
// Reconnect loads back to upper_insert_pt. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1506 |
for (uint i = 0; i < memops.size(); i++) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1507 |
Node *ld = memops.at(i); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1508 |
if (ld->in(MemNode::Memory) != upper_insert_pt) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1509 |
_igvn.replace_input_of(ld, MemNode::Memory, upper_insert_pt); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1510 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1511 |
} |
2334 | 1512 |
} else if (pk->at(0)->is_Load()) { //load |
3799 | 1513 |
// all loads in the pack should have the same memory state. By default, |
1514 |
// we use the memory state of the last load. However, if any load could |
|
1515 |
// not be moved down due to the dependence constraint, we use the memory |
|
1516 |
// state of the first load. |
|
1517 |
Node* last_mem = executed_last(pk)->in(MemNode::Memory); |
|
1518 |
Node* first_mem = executed_first(pk)->in(MemNode::Memory); |
|
1519 |
bool schedule_last = true; |
|
1520 |
for (uint i = 0; i < pk->size(); i++) { |
|
1521 |
Node* ld = pk->at(i); |
|
1522 |
for (Node* current = last_mem; current != ld->in(MemNode::Memory); |
|
1523 |
current=current->in(MemNode::Memory)) { |
|
1524 |
assert(current != first_mem, "corrupted memory graph"); |
|
1525 |
if(current->is_Mem() && !independent(current, ld)){ |
|
1526 |
schedule_last = false; // a later store depends on this load |
|
1527 |
break; |
|
1528 |
} |
|
1529 |
} |
|
1530 |
} |
|
1531 |
||
1532 |
Node* mem_input = schedule_last ? last_mem : first_mem; |
|
1533 |
_igvn.hash_delete(mem_input); |
|
1534 |
// Give each load the same memory state |
|
1 | 1535 |
for (uint i = 0; i < pk->size(); i++) { |
1536 |
LoadNode* ld = pk->at(i)->as_Load(); |
|
12958
009b6c9586d8
7173340: C2: code cleanup: use PhaseIterGVN::replace_edge(Node*, int, Node*) where applicable
kvn
parents:
12594
diff
changeset
|
1537 |
_igvn.replace_input_of(ld, MemNode::Memory, mem_input); |
1 | 1538 |
} |
1539 |
} |
|
1540 |
} |
|
1541 |
||
1542 |
//------------------------------output--------------------------- |
|
1543 |
// Convert packs into vector node operations |
|
1544 |
void SuperWord::output() { |
|
1545 |
if (_packset.length() == 0) return; |
|
1546 |
||
9101 | 1547 |
#ifndef PRODUCT |
1548 |
if (TraceLoopOpts) { |
|
1549 |
tty->print("SuperWord "); |
|
1550 |
lpt()->dump_head(); |
|
1551 |
} |
|
1552 |
#endif |
|
1553 |
||
1 | 1554 |
// MUST ENSURE main loop's initial value is properly aligned: |
1555 |
// (iv_initial_value + min_iv_offset) % vector_width_in_bytes() == 0 |
|
1556 |
||
1557 |
align_initial_loop_index(align_to_ref()); |
|
1558 |
||
1559 |
// Insert extract (unpack) operations for scalar uses |
|
1560 |
for (int i = 0; i < _packset.length(); i++) { |
|
1561 |
insert_extracts(_packset.at(i)); |
|
1562 |
} |
|
1563 |
||
13883
6979b9850feb
7196199: java/text/Bidi/Bug6665028.java failed: Bidi run count incorrect
kvn
parents:
13490
diff
changeset
|
1564 |
Compile* C = _phase->C; |
6979b9850feb
7196199: java/text/Bidi/Bug6665028.java failed: Bidi run count incorrect
kvn
parents:
13490
diff
changeset
|
1565 |
uint max_vlen_in_bytes = 0; |
1 | 1566 |
for (int i = 0; i < _block.length(); i++) { |
1567 |
Node* n = _block.at(i); |
|
1568 |
Node_List* p = my_pack(n); |
|
1569 |
if (p && n == executed_last(p)) { |
|
1570 |
uint vlen = p->size(); |
|
13883
6979b9850feb
7196199: java/text/Bidi/Bug6665028.java failed: Bidi run count incorrect
kvn
parents:
13490
diff
changeset
|
1571 |
uint vlen_in_bytes = 0; |
1 | 1572 |
Node* vn = NULL; |
1573 |
Node* low_adr = p->at(0); |
|
1574 |
Node* first = executed_first(p); |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1575 |
int opc = n->Opcode(); |
1 | 1576 |
if (n->is_Load()) { |
1577 |
Node* ctl = n->in(MemNode::Control); |
|
1578 |
Node* mem = first->in(MemNode::Memory); |
|
25932
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
25930
diff
changeset
|
1579 |
SWPointer p1(n->as_Mem(), this); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
25930
diff
changeset
|
1580 |
// Identify the memory dependency for the new loadVector node by |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
25930
diff
changeset
|
1581 |
// walking up through memory chain. |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
25930
diff
changeset
|
1582 |
// This is done to give flexibility to the new loadVector node so that |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
25930
diff
changeset
|
1583 |
// it can move above independent storeVector nodes. |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
25930
diff
changeset
|
1584 |
while (mem->is_StoreVector()) { |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
25930
diff
changeset
|
1585 |
SWPointer p2(mem->as_Mem(), this); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
25930
diff
changeset
|
1586 |
int cmp = p1.cmp(p2); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
25930
diff
changeset
|
1587 |
if (SWPointer::not_equal(cmp) || !SWPointer::comparable(cmp)) { |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
25930
diff
changeset
|
1588 |
mem = mem->in(MemNode::Memory); |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
25930
diff
changeset
|
1589 |
} else { |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
25930
diff
changeset
|
1590 |
break; // dependent memory |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
25930
diff
changeset
|
1591 |
} |
15d133edd8f6
8052081: Optimize generated by C2 code for Intel's Atom processor
kvn
parents:
25930
diff
changeset
|
1592 |
} |
1 | 1593 |
Node* adr = low_adr->in(MemNode::Address); |
1594 |
const TypePtr* atyp = n->adr_type(); |
|
25930 | 1595 |
vn = LoadVectorNode::make(opc, ctl, mem, adr, atyp, vlen, velt_basic_type(n)); |
13883
6979b9850feb
7196199: java/text/Bidi/Bug6665028.java failed: Bidi run count incorrect
kvn
parents:
13490
diff
changeset
|
1596 |
vlen_in_bytes = vn->as_LoadVector()->memory_size(); |
1 | 1597 |
} else if (n->is_Store()) { |
1598 |
// Promote value to be stored to vector |
|
10255 | 1599 |
Node* val = vector_opd(p, MemNode::ValueIn); |
1 | 1600 |
Node* ctl = n->in(MemNode::Control); |
1601 |
Node* mem = first->in(MemNode::Memory); |
|
1602 |
Node* adr = low_adr->in(MemNode::Address); |
|
1603 |
const TypePtr* atyp = n->adr_type(); |
|
25930 | 1604 |
vn = StoreVectorNode::make(opc, ctl, mem, adr, atyp, val, vlen); |
13883
6979b9850feb
7196199: java/text/Bidi/Bug6665028.java failed: Bidi run count incorrect
kvn
parents:
13490
diff
changeset
|
1605 |
vlen_in_bytes = vn->as_StoreVector()->memory_size(); |
1 | 1606 |
} else if (n->req() == 3) { |
1607 |
// Promote operands to vector |
|
30211 | 1608 |
Node* in1 = NULL; |
1609 |
bool node_isa_reduction = n->is_reduction(); |
|
1610 |
if (node_isa_reduction) { |
|
1611 |
// the input to the first reduction operation is retained |
|
1612 |
in1 = low_adr->in(1); |
|
1613 |
} else { |
|
1614 |
in1 = vector_opd(p, 1); |
|
1615 |
} |
|
1 | 1616 |
Node* in2 = vector_opd(p, 2); |
30211 | 1617 |
if (VectorNode::is_invariant_vector(in1) && (node_isa_reduction == false) && (n->is_Add() || n->is_Mul())) { |
13485
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
1618 |
// Move invariant vector input into second position to avoid register spilling. |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
1619 |
Node* tmp = in1; |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
1620 |
in1 = in2; |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
1621 |
in2 = tmp; |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
1622 |
} |
30211 | 1623 |
if (node_isa_reduction) { |
1624 |
const Type *arith_type = n->bottom_type(); |
|
1625 |
vn = ReductionNode::make(opc, NULL, in1, in2, arith_type->basic_type()); |
|
1626 |
if (in2->is_Load()) { |
|
1627 |
vlen_in_bytes = in2->as_LoadVector()->memory_size(); |
|
1628 |
} else { |
|
1629 |
vlen_in_bytes = in2->as_Vector()->length_in_bytes(); |
|
1630 |
} |
|
1631 |
} else { |
|
1632 |
vn = VectorNode::make(opc, in1, in2, vlen, velt_basic_type(n)); |
|
1633 |
vlen_in_bytes = vn->as_Vector()->length_in_bytes(); |
|
1634 |
} |
|
1 | 1635 |
} else { |
1636 |
ShouldNotReachHere(); |
|
1637 |
} |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1638 |
assert(vn != NULL, "sanity"); |
13894 | 1639 |
_igvn.register_new_node_with_optimizer(vn); |
1 | 1640 |
_phase->set_ctrl(vn, _phase->get_ctrl(p->at(0))); |
1641 |
for (uint j = 0; j < p->size(); j++) { |
|
1642 |
Node* pm = p->at(j); |
|
5901
c046f8e9c52b
6677629: PhaseIterGVN::subsume_node() should call hash_delete() and add_users_to_worklist()
kvn
parents:
5708
diff
changeset
|
1643 |
_igvn.replace_node(pm, vn); |
1 | 1644 |
} |
1645 |
_igvn._worklist.push(vn); |
|
13883
6979b9850feb
7196199: java/text/Bidi/Bug6665028.java failed: Bidi run count incorrect
kvn
parents:
13490
diff
changeset
|
1646 |
|
6979b9850feb
7196199: java/text/Bidi/Bug6665028.java failed: Bidi run count incorrect
kvn
parents:
13490
diff
changeset
|
1647 |
if (vlen_in_bytes > max_vlen_in_bytes) { |
6979b9850feb
7196199: java/text/Bidi/Bug6665028.java failed: Bidi run count incorrect
kvn
parents:
13490
diff
changeset
|
1648 |
max_vlen_in_bytes = vlen_in_bytes; |
6979b9850feb
7196199: java/text/Bidi/Bug6665028.java failed: Bidi run count incorrect
kvn
parents:
13490
diff
changeset
|
1649 |
} |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1650 |
#ifdef ASSERT |
13108
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
1651 |
if (TraceNewVectors) { |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1652 |
tty->print("new Vector node: "); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1653 |
vn->dump(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1654 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1655 |
#endif |
1 | 1656 |
} |
1657 |
} |
|
13883
6979b9850feb
7196199: java/text/Bidi/Bug6665028.java failed: Bidi run count incorrect
kvn
parents:
13490
diff
changeset
|
1658 |
C->set_max_vector_size(max_vlen_in_bytes); |
1 | 1659 |
} |
1660 |
||
1661 |
//------------------------------vector_opd--------------------------- |
|
1662 |
// Create a vector operand for the nodes in pack p for operand: in(opd_idx) |
|
10255 | 1663 |
Node* SuperWord::vector_opd(Node_List* p, int opd_idx) { |
1 | 1664 |
Node* p0 = p->at(0); |
1665 |
uint vlen = p->size(); |
|
1666 |
Node* opd = p0->in(opd_idx); |
|
1667 |
||
13490
d19348851d2e
7192963: assert(_in[req-1] == this) failed: Must pass arg count to 'new'
kvn
parents:
13488
diff
changeset
|
1668 |
if (same_inputs(p, opd_idx)) { |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1669 |
if (opd->is_Vector() || opd->is_LoadVector()) { |
13488 | 1670 |
assert(((opd_idx != 2) || !VectorNode::is_shift(p0)), "shift's count can't be vector"); |
10255 | 1671 |
return opd; // input is matching vector |
1 | 1672 |
} |
13485
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
1673 |
if ((opd_idx == 2) && VectorNode::is_shift(p0)) { |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
1674 |
Compile* C = _phase->C; |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
1675 |
Node* cnt = opd; |
13930 | 1676 |
// Vector instructions do not mask shift count, do it here. |
13485
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
1677 |
juint mask = (p0->bottom_type() == TypeInt::INT) ? (BitsPerInt - 1) : (BitsPerLong - 1); |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
1678 |
const TypeInt* t = opd->find_int_type(); |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
1679 |
if (t != NULL && t->is_con()) { |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
1680 |
juint shift = t->get_con(); |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
1681 |
if (shift > mask) { // Unsigned cmp |
25930 | 1682 |
cnt = ConNode::make(TypeInt::make(shift & mask)); |
13485
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
1683 |
} |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
1684 |
} else { |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
1685 |
if (t == NULL || t->_lo < 0 || t->_hi > (int)mask) { |
25930 | 1686 |
cnt = ConNode::make(TypeInt::make(mask)); |
13894 | 1687 |
_igvn.register_new_node_with_optimizer(cnt); |
24923
9631f7d691dc
8034812: remove IDX_INIT macro hack in Node class
thartmann
parents:
24090
diff
changeset
|
1688 |
cnt = new AndINode(opd, cnt); |
13894 | 1689 |
_igvn.register_new_node_with_optimizer(cnt); |
13485
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
1690 |
_phase->set_ctrl(cnt, _phase->get_ctrl(opd)); |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
1691 |
} |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
1692 |
assert(opd->bottom_type()->isa_int(), "int type only"); |
13930 | 1693 |
// Move non constant shift count into vector register. |
25930 | 1694 |
cnt = VectorNode::shift_count(p0, cnt, vlen, velt_basic_type(p0)); |
13485
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
1695 |
} |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
1696 |
if (cnt != opd) { |
13894 | 1697 |
_igvn.register_new_node_with_optimizer(cnt); |
13485
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
1698 |
_phase->set_ctrl(cnt, _phase->get_ctrl(opd)); |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
1699 |
} |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
1700 |
return cnt; |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
1701 |
} |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1702 |
assert(!opd->is_StoreVector(), "such vector is not expected here"); |
12594 | 1703 |
// Convert scalar input to vector with the same number of elements as |
1704 |
// p0's vector. Use p0's type because size of operand's container in |
|
1705 |
// vector should match p0's size regardless operand's size. |
|
1706 |
const Type* p0_t = velt_type(p0); |
|
25930 | 1707 |
VectorNode* vn = VectorNode::scalar2vector(opd, vlen, p0_t); |
1 | 1708 |
|
13894 | 1709 |
_igvn.register_new_node_with_optimizer(vn); |
1 | 1710 |
_phase->set_ctrl(vn, _phase->get_ctrl(opd)); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1711 |
#ifdef ASSERT |
13108
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
1712 |
if (TraceNewVectors) { |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1713 |
tty->print("new Vector node: "); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1714 |
vn->dump(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1715 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1716 |
#endif |
1 | 1717 |
return vn; |
1718 |
} |
|
1719 |
||
1720 |
// Insert pack operation |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1721 |
BasicType bt = velt_basic_type(p0); |
25930 | 1722 |
PackNode* pk = PackNode::make(opd, vlen, bt); |
12594 | 1723 |
DEBUG_ONLY( const BasicType opd_bt = opd->bottom_type()->basic_type(); ) |
1 | 1724 |
|
1725 |
for (uint i = 1; i < vlen; i++) { |
|
1726 |
Node* pi = p->at(i); |
|
1727 |
Node* in = pi->in(opd_idx); |
|
1728 |
assert(my_pack(in) == NULL, "Should already have been unpacked"); |
|
12594 | 1729 |
assert(opd_bt == in->bottom_type()->basic_type(), "all same type"); |
13490
d19348851d2e
7192963: assert(_in[req-1] == this) failed: Must pass arg count to 'new'
kvn
parents:
13488
diff
changeset
|
1730 |
pk->add_opd(in); |
1 | 1731 |
} |
13894 | 1732 |
_igvn.register_new_node_with_optimizer(pk); |
1 | 1733 |
_phase->set_ctrl(pk, _phase->get_ctrl(opd)); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1734 |
#ifdef ASSERT |
13883
6979b9850feb
7196199: java/text/Bidi/Bug6665028.java failed: Bidi run count incorrect
kvn
parents:
13490
diff
changeset
|
1735 |
if (TraceNewVectors) { |
6979b9850feb
7196199: java/text/Bidi/Bug6665028.java failed: Bidi run count incorrect
kvn
parents:
13490
diff
changeset
|
1736 |
tty->print("new Vector node: "); |
6979b9850feb
7196199: java/text/Bidi/Bug6665028.java failed: Bidi run count incorrect
kvn
parents:
13490
diff
changeset
|
1737 |
pk->dump(); |
6979b9850feb
7196199: java/text/Bidi/Bug6665028.java failed: Bidi run count incorrect
kvn
parents:
13490
diff
changeset
|
1738 |
} |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1739 |
#endif |
1 | 1740 |
return pk; |
1741 |
} |
|
1742 |
||
1743 |
//------------------------------insert_extracts--------------------------- |
|
1744 |
// If a use of pack p is not a vector use, then replace the |
|
1745 |
// use with an extract operation. |
|
1746 |
void SuperWord::insert_extracts(Node_List* p) { |
|
1747 |
if (p->at(0)->is_Store()) return; |
|
1748 |
assert(_n_idx_list.is_empty(), "empty (node,index) list"); |
|
1749 |
||
1750 |
// Inspect each use of each pack member. For each use that is |
|
1751 |
// not a vector use, replace the use with an extract operation. |
|
1752 |
||
1753 |
for (uint i = 0; i < p->size(); i++) { |
|
1754 |
Node* def = p->at(i); |
|
1755 |
for (DUIterator_Fast jmax, j = def->fast_outs(jmax); j < jmax; j++) { |
|
1756 |
Node* use = def->fast_out(j); |
|
1757 |
for (uint k = 0; k < use->req(); k++) { |
|
1758 |
Node* n = use->in(k); |
|
1759 |
if (def == n) { |
|
1760 |
if (!is_vector_use(use, k)) { |
|
1761 |
_n_idx_list.push(use, k); |
|
1762 |
} |
|
1763 |
} |
|
1764 |
} |
|
1765 |
} |
|
1766 |
} |
|
1767 |
||
1768 |
while (_n_idx_list.is_nonempty()) { |
|
1769 |
Node* use = _n_idx_list.node(); |
|
1770 |
int idx = _n_idx_list.index(); |
|
1771 |
_n_idx_list.pop(); |
|
1772 |
Node* def = use->in(idx); |
|
1773 |
||
30211 | 1774 |
if (def->is_reduction()) continue; |
1775 |
||
1 | 1776 |
// Insert extract operation |
1777 |
_igvn.hash_delete(def); |
|
1778 |
int def_pos = alignment(def) / data_size(def); |
|
1779 |
||
25930 | 1780 |
Node* ex = ExtractNode::make(def, def_pos, velt_basic_type(def)); |
13894 | 1781 |
_igvn.register_new_node_with_optimizer(ex); |
1 | 1782 |
_phase->set_ctrl(ex, _phase->get_ctrl(def)); |
12958
009b6c9586d8
7173340: C2: code cleanup: use PhaseIterGVN::replace_edge(Node*, int, Node*) where applicable
kvn
parents:
12594
diff
changeset
|
1783 |
_igvn.replace_input_of(use, idx, ex); |
1 | 1784 |
_igvn._worklist.push(def); |
1785 |
||
1786 |
bb_insert_after(ex, bb_idx(def)); |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1787 |
set_velt_type(ex, velt_type(def)); |
1 | 1788 |
} |
1789 |
} |
|
1790 |
||
1791 |
//------------------------------is_vector_use--------------------------- |
|
1792 |
// Is use->in(u_idx) a vector use? |
|
1793 |
bool SuperWord::is_vector_use(Node* use, int u_idx) { |
|
1794 |
Node_List* u_pk = my_pack(use); |
|
1795 |
if (u_pk == NULL) return false; |
|
30211 | 1796 |
if (use->is_reduction()) return true; |
1 | 1797 |
Node* def = use->in(u_idx); |
1798 |
Node_List* d_pk = my_pack(def); |
|
1799 |
if (d_pk == NULL) { |
|
1800 |
// check for scalar promotion |
|
1801 |
Node* n = u_pk->at(0)->in(u_idx); |
|
1802 |
for (uint i = 1; i < u_pk->size(); i++) { |
|
1803 |
if (u_pk->at(i)->in(u_idx) != n) return false; |
|
1804 |
} |
|
1805 |
return true; |
|
1806 |
} |
|
1807 |
if (u_pk->size() != d_pk->size()) |
|
1808 |
return false; |
|
1809 |
for (uint i = 0; i < u_pk->size(); i++) { |
|
1810 |
Node* ui = u_pk->at(i); |
|
1811 |
Node* di = d_pk->at(i); |
|
1812 |
if (ui->in(u_idx) != di || alignment(ui) != alignment(di)) |
|
1813 |
return false; |
|
1814 |
} |
|
1815 |
return true; |
|
1816 |
} |
|
1817 |
||
1818 |
//------------------------------construct_bb--------------------------- |
|
1819 |
// Construct reverse postorder list of block members |
|
15755
c54cbfb4b37f
8004867: VM crashing with assert "share/vm/opto/node.hpp:357 - assert(i < _max) failed: oob"
kvn
parents:
14134
diff
changeset
|
1820 |
bool SuperWord::construct_bb() { |
1 | 1821 |
Node* entry = bb(); |
1822 |
||
1823 |
assert(_stk.length() == 0, "stk is empty"); |
|
1824 |
assert(_block.length() == 0, "block is empty"); |
|
1825 |
assert(_data_entry.length() == 0, "data_entry is empty"); |
|
1826 |
assert(_mem_slice_head.length() == 0, "mem_slice_head is empty"); |
|
1827 |
assert(_mem_slice_tail.length() == 0, "mem_slice_tail is empty"); |
|
1828 |
||
1829 |
// Find non-control nodes with no inputs from within block, |
|
1830 |
// create a temporary map from node _idx to bb_idx for use |
|
1831 |
// by the visited and post_visited sets, |
|
1832 |
// and count number of nodes in block. |
|
1833 |
int bb_ct = 0; |
|
30211 | 1834 |
for (uint i = 0; i < lpt()->_body.size(); i++) { |
1 | 1835 |
Node *n = lpt()->_body.at(i); |
1836 |
set_bb_idx(n, i); // Create a temporary map |
|
1837 |
if (in_bb(n)) { |
|
15755
c54cbfb4b37f
8004867: VM crashing with assert "share/vm/opto/node.hpp:357 - assert(i < _max) failed: oob"
kvn
parents:
14134
diff
changeset
|
1838 |
if (n->is_LoadStore() || n->is_MergeMem() || |
c54cbfb4b37f
8004867: VM crashing with assert "share/vm/opto/node.hpp:357 - assert(i < _max) failed: oob"
kvn
parents:
14134
diff
changeset
|
1839 |
(n->is_Proj() && !n->as_Proj()->is_CFG())) { |
c54cbfb4b37f
8004867: VM crashing with assert "share/vm/opto/node.hpp:357 - assert(i < _max) failed: oob"
kvn
parents:
14134
diff
changeset
|
1840 |
// Bailout if the loop has LoadStore, MergeMem or data Proj |
c54cbfb4b37f
8004867: VM crashing with assert "share/vm/opto/node.hpp:357 - assert(i < _max) failed: oob"
kvn
parents:
14134
diff
changeset
|
1841 |
// nodes. Superword optimization does not work with them. |
c54cbfb4b37f
8004867: VM crashing with assert "share/vm/opto/node.hpp:357 - assert(i < _max) failed: oob"
kvn
parents:
14134
diff
changeset
|
1842 |
return false; |
c54cbfb4b37f
8004867: VM crashing with assert "share/vm/opto/node.hpp:357 - assert(i < _max) failed: oob"
kvn
parents:
14134
diff
changeset
|
1843 |
} |
1 | 1844 |
bb_ct++; |
1845 |
if (!n->is_CFG()) { |
|
1846 |
bool found = false; |
|
1847 |
for (uint j = 0; j < n->req(); j++) { |
|
1848 |
Node* def = n->in(j); |
|
1849 |
if (def && in_bb(def)) { |
|
1850 |
found = true; |
|
1851 |
break; |
|
1852 |
} |
|
1853 |
} |
|
1854 |
if (!found) { |
|
1855 |
assert(n != entry, "can't be entry"); |
|
1856 |
_data_entry.push(n); |
|
1857 |
} |
|
1858 |
} |
|
1859 |
} |
|
1860 |
} |
|
1861 |
||
1862 |
// Find memory slices (head and tail) |
|
1863 |
for (DUIterator_Fast imax, i = lp()->fast_outs(imax); i < imax; i++) { |
|
1864 |
Node *n = lp()->fast_out(i); |
|
1865 |
if (in_bb(n) && (n->is_Phi() && n->bottom_type() == Type::MEMORY)) { |
|
1866 |
Node* n_tail = n->in(LoopNode::LoopBackControl); |
|
961
7fb3b13d4205
6726999: nsk/stress/jck12a/jck12a010 assert(n != null,"Bad immediate dominator info.")
kvn
parents:
781
diff
changeset
|
1867 |
if (n_tail != n->in(LoopNode::EntryControl)) { |
15755
c54cbfb4b37f
8004867: VM crashing with assert "share/vm/opto/node.hpp:357 - assert(i < _max) failed: oob"
kvn
parents:
14134
diff
changeset
|
1868 |
if (!n_tail->is_Mem()) { |
c54cbfb4b37f
8004867: VM crashing with assert "share/vm/opto/node.hpp:357 - assert(i < _max) failed: oob"
kvn
parents:
14134
diff
changeset
|
1869 |
assert(n_tail->is_Mem(), err_msg_res("unexpected node for memory slice: %s", n_tail->Name())); |
c54cbfb4b37f
8004867: VM crashing with assert "share/vm/opto/node.hpp:357 - assert(i < _max) failed: oob"
kvn
parents:
14134
diff
changeset
|
1870 |
return false; // Bailout |
c54cbfb4b37f
8004867: VM crashing with assert "share/vm/opto/node.hpp:357 - assert(i < _max) failed: oob"
kvn
parents:
14134
diff
changeset
|
1871 |
} |
961
7fb3b13d4205
6726999: nsk/stress/jck12a/jck12a010 assert(n != null,"Bad immediate dominator info.")
kvn
parents:
781
diff
changeset
|
1872 |
_mem_slice_head.push(n); |
7fb3b13d4205
6726999: nsk/stress/jck12a/jck12a010 assert(n != null,"Bad immediate dominator info.")
kvn
parents:
781
diff
changeset
|
1873 |
_mem_slice_tail.push(n_tail); |
7fb3b13d4205
6726999: nsk/stress/jck12a/jck12a010 assert(n != null,"Bad immediate dominator info.")
kvn
parents:
781
diff
changeset
|
1874 |
} |
1 | 1875 |
} |
1876 |
} |
|
1877 |
||
1878 |
// Create an RPO list of nodes in block |
|
1879 |
||
1880 |
visited_clear(); |
|
1881 |
post_visited_clear(); |
|
1882 |
||
1883 |
// Push all non-control nodes with no inputs from within block, then control entry |
|
1884 |
for (int j = 0; j < _data_entry.length(); j++) { |
|
1885 |
Node* n = _data_entry.at(j); |
|
1886 |
visited_set(n); |
|
1887 |
_stk.push(n); |
|
1888 |
} |
|
1889 |
visited_set(entry); |
|
1890 |
_stk.push(entry); |
|
1891 |
||
1892 |
// Do a depth first walk over out edges |
|
1893 |
int rpo_idx = bb_ct - 1; |
|
1894 |
int size; |
|
30211 | 1895 |
int reduction_uses = 0; |
1 | 1896 |
while ((size = _stk.length()) > 0) { |
1897 |
Node* n = _stk.top(); // Leave node on stack |
|
1898 |
if (!visited_test_set(n)) { |
|
1899 |
// forward arc in graph |
|
1900 |
} else if (!post_visited_test(n)) { |
|
1901 |
// cross or back arc |
|
1902 |
for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { |
|
1903 |
Node *use = n->fast_out(i); |
|
1904 |
if (in_bb(use) && !visited_test(use) && |
|
1905 |
// Don't go around backedge |
|
1906 |
(!use->is_Phi() || n == entry)) { |
|
30211 | 1907 |
if (use->is_reduction()) { |
1908 |
// First see if we can map the reduction on the given system we are on, then |
|
1909 |
// make a data entry operation for each reduction we see. |
|
1910 |
BasicType bt = use->bottom_type()->basic_type(); |
|
1911 |
if (ReductionNode::implemented(use->Opcode(), Matcher::min_vector_size(bt), bt)) { |
|
1912 |
reduction_uses++; |
|
1913 |
} |
|
1914 |
} |
|
1 | 1915 |
_stk.push(use); |
1916 |
} |
|
1917 |
} |
|
1918 |
if (_stk.length() == size) { |
|
1919 |
// There were no additional uses, post visit node now |
|
1920 |
_stk.pop(); // Remove node from stack |
|
1921 |
assert(rpo_idx >= 0, ""); |
|
1922 |
_block.at_put_grow(rpo_idx, n); |
|
1923 |
rpo_idx--; |
|
1924 |
post_visited_set(n); |
|
1925 |
assert(rpo_idx >= 0 || _stk.is_empty(), ""); |
|
1926 |
} |
|
1927 |
} else { |
|
1928 |
_stk.pop(); // Remove post-visited node from stack |
|
1929 |
} |
|
1930 |
} |
|
1931 |
||
1932 |
// Create real map of block indices for nodes |
|
1933 |
for (int j = 0; j < _block.length(); j++) { |
|
1934 |
Node* n = _block.at(j); |
|
1935 |
set_bb_idx(n, j); |
|
1936 |
} |
|
1937 |
||
30211 | 1938 |
// Ensure extra info is allocated. |
1939 |
initialize_bb(); |
|
1 | 1940 |
|
1941 |
#ifndef PRODUCT |
|
1942 |
if (TraceSuperWord) { |
|
1943 |
print_bb(); |
|
1944 |
tty->print_cr("\ndata entry nodes: %s", _data_entry.length() > 0 ? "" : "NONE"); |
|
1945 |
for (int m = 0; m < _data_entry.length(); m++) { |
|
1946 |
tty->print("%3d ", m); |
|
1947 |
_data_entry.at(m)->dump(); |
|
1948 |
} |
|
1949 |
tty->print_cr("\nmemory slices: %s", _mem_slice_head.length() > 0 ? "" : "NONE"); |
|
1950 |
for (int m = 0; m < _mem_slice_head.length(); m++) { |
|
1951 |
tty->print("%3d ", m); _mem_slice_head.at(m)->dump(); |
|
1952 |
tty->print(" "); _mem_slice_tail.at(m)->dump(); |
|
1953 |
} |
|
1954 |
} |
|
1955 |
#endif |
|
1956 |
assert(rpo_idx == -1 && bb_ct == _block.length(), "all block members found"); |
|
30211 | 1957 |
return (_mem_slice_head.length() > 0) || (reduction_uses > 0) || (_data_entry.length() > 0); |
1 | 1958 |
} |
1959 |
||
1960 |
//------------------------------initialize_bb--------------------------- |
|
1961 |
// Initialize per node info |
|
1962 |
void SuperWord::initialize_bb() { |
|
1963 |
Node* last = _block.at(_block.length() - 1); |
|
1964 |
grow_node_info(bb_idx(last)); |
|
1965 |
} |
|
1966 |
||
1967 |
//------------------------------bb_insert_after--------------------------- |
|
1968 |
// Insert n into block after pos |
|
1969 |
void SuperWord::bb_insert_after(Node* n, int pos) { |
|
1970 |
int n_pos = pos + 1; |
|
1971 |
// Make room |
|
1972 |
for (int i = _block.length() - 1; i >= n_pos; i--) { |
|
1973 |
_block.at_put_grow(i+1, _block.at(i)); |
|
1974 |
} |
|
1975 |
for (int j = _node_info.length() - 1; j >= n_pos; j--) { |
|
1976 |
_node_info.at_put_grow(j+1, _node_info.at(j)); |
|
1977 |
} |
|
1978 |
// Set value |
|
1979 |
_block.at_put_grow(n_pos, n); |
|
1980 |
_node_info.at_put_grow(n_pos, SWNodeInfo::initial); |
|
1981 |
// Adjust map from node->_idx to _block index |
|
1982 |
for (int i = n_pos; i < _block.length(); i++) { |
|
1983 |
set_bb_idx(_block.at(i), i); |
|
1984 |
} |
|
1985 |
} |
|
1986 |
||
1987 |
//------------------------------compute_max_depth--------------------------- |
|
1988 |
// Compute max depth for expressions from beginning of block |
|
1989 |
// Use to prune search paths during test for independence. |
|
1990 |
void SuperWord::compute_max_depth() { |
|
1991 |
int ct = 0; |
|
1992 |
bool again; |
|
1993 |
do { |
|
1994 |
again = false; |
|
1995 |
for (int i = 0; i < _block.length(); i++) { |
|
1996 |
Node* n = _block.at(i); |
|
1997 |
if (!n->is_Phi()) { |
|
1998 |
int d_orig = depth(n); |
|
1999 |
int d_in = 0; |
|
2000 |
for (DepPreds preds(n, _dg); !preds.done(); preds.next()) { |
|
2001 |
Node* pred = preds.current(); |
|
2002 |
if (in_bb(pred)) { |
|
2003 |
d_in = MAX2(d_in, depth(pred)); |
|
2004 |
} |
|
2005 |
} |
|
2006 |
if (d_in + 1 != d_orig) { |
|
2007 |
set_depth(n, d_in + 1); |
|
2008 |
again = true; |
|
2009 |
} |
|
2010 |
} |
|
2011 |
} |
|
2012 |
ct++; |
|
2013 |
} while (again); |
|
2014 |
#ifndef PRODUCT |
|
2015 |
if (TraceSuperWord && Verbose) |
|
2016 |
tty->print_cr("compute_max_depth iterated: %d times", ct); |
|
2017 |
#endif |
|
2018 |
} |
|
2019 |
||
2020 |
//-------------------------compute_vector_element_type----------------------- |
|
2021 |
// Compute necessary vector element type for expressions |
|
2022 |
// This propagates backwards a narrower integer type when the |
|
2023 |
// upper bits of the value are not needed. |
|
2024 |
// Example: char a,b,c; a = b + c; |
|
2025 |
// Normally the type of the add is integer, but for packed character |
|
2026 |
// operations the type of the add needs to be char. |
|
2027 |
void SuperWord::compute_vector_element_type() { |
|
2028 |
#ifndef PRODUCT |
|
2029 |
if (TraceSuperWord && Verbose) |
|
2030 |
tty->print_cr("\ncompute_velt_type:"); |
|
2031 |
#endif |
|
2032 |
||
2033 |
// Initial type |
|
2034 |
for (int i = 0; i < _block.length(); i++) { |
|
2035 |
Node* n = _block.at(i); |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2036 |
set_velt_type(n, container_type(n)); |
1 | 2037 |
} |
2038 |
||
14131
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2039 |
// Propagate integer narrowed type backwards through operations |
1 | 2040 |
// that don't depend on higher order bits |
2041 |
for (int i = _block.length() - 1; i >= 0; i--) { |
|
2042 |
Node* n = _block.at(i); |
|
2043 |
// Only integer types need be examined |
|
14131
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2044 |
const Type* vtn = velt_type(n); |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2045 |
if (vtn->basic_type() == T_INT) { |
1 | 2046 |
uint start, end; |
13490
d19348851d2e
7192963: assert(_in[req-1] == this) failed: Must pass arg count to 'new'
kvn
parents:
13488
diff
changeset
|
2047 |
VectorNode::vector_operands(n, &start, &end); |
1 | 2048 |
|
2049 |
for (uint j = start; j < end; j++) { |
|
2050 |
Node* in = n->in(j); |
|
13485
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
2051 |
// Don't propagate through a memory |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
2052 |
if (!in->is_Mem() && in_bb(in) && velt_type(in)->basic_type() == T_INT && |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
2053 |
data_size(n) < data_size(in)) { |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
2054 |
bool same_type = true; |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
2055 |
for (DUIterator_Fast kmax, k = in->fast_outs(kmax); k < kmax; k++) { |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
2056 |
Node *use = in->fast_out(k); |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
2057 |
if (!in_bb(use) || !same_velt_type(use, n)) { |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
2058 |
same_type = false; |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
2059 |
break; |
1 | 2060 |
} |
13485
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
2061 |
} |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
2062 |
if (same_type) { |
14131
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2063 |
// For right shifts of small integer types (bool, byte, char, short) |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2064 |
// we need precise information about sign-ness. Only Load nodes have |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2065 |
// this information because Store nodes are the same for signed and |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2066 |
// unsigned values. And any arithmetic operation after a load may |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2067 |
// expand a value to signed Int so such right shifts can't be used |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2068 |
// because vector elements do not have upper bits of Int. |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2069 |
const Type* vt = vtn; |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2070 |
if (VectorNode::is_shift(in)) { |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2071 |
Node* load = in->in(1); |
14134 | 2072 |
if (load->is_Load() && in_bb(load) && (velt_type(load)->basic_type() == T_INT)) { |
14131
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2073 |
vt = velt_type(load); |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2074 |
} else if (in->Opcode() != Op_LShiftI) { |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2075 |
// Widen type to Int to avoid creation of right shift vector |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2076 |
// (align + data_size(s1) check in stmts_can_pack() will fail). |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2077 |
// Note, left shifts work regardless type. |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2078 |
vt = TypeInt::INT; |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2079 |
} |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2080 |
} |
13485
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
2081 |
set_velt_type(in, vt); |
1 | 2082 |
} |
2083 |
} |
|
2084 |
} |
|
2085 |
} |
|
2086 |
} |
|
2087 |
#ifndef PRODUCT |
|
2088 |
if (TraceSuperWord && Verbose) { |
|
2089 |
for (int i = 0; i < _block.length(); i++) { |
|
2090 |
Node* n = _block.at(i); |
|
2091 |
velt_type(n)->dump(); |
|
2092 |
tty->print("\t"); |
|
2093 |
n->dump(); |
|
2094 |
} |
|
2095 |
} |
|
2096 |
#endif |
|
2097 |
} |
|
2098 |
||
2099 |
//------------------------------memory_alignment--------------------------- |
|
2100 |
// Alignment within a vector memory reference |
|
13885 | 2101 |
int SuperWord::memory_alignment(MemNode* s, int iv_adjust) { |
1 | 2102 |
SWPointer p(s, this); |
2103 |
if (!p.valid()) { |
|
2104 |
return bottom_align; |
|
2105 |
} |
|
13108
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
2106 |
int vw = vector_width_in_bytes(s); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2107 |
if (vw < 2) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2108 |
return bottom_align; // No vectors for this type |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2109 |
} |
1 | 2110 |
int offset = p.offset_in_bytes(); |
13885 | 2111 |
offset += iv_adjust*p.memory_size(); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2112 |
int off_rem = offset % vw; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2113 |
int off_mod = off_rem >= 0 ? off_rem : off_rem + vw; |
1 | 2114 |
return off_mod; |
2115 |
} |
|
2116 |
||
2117 |
//---------------------------container_type--------------------------- |
|
2118 |
// Smallest type containing range of values |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2119 |
const Type* SuperWord::container_type(Node* n) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2120 |
if (n->is_Mem()) { |
14131
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2121 |
BasicType bt = n->as_Mem()->memory_type(); |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2122 |
if (n->is_Store() && (bt == T_CHAR)) { |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2123 |
// Use T_SHORT type instead of T_CHAR for stored values because any |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2124 |
// preceding arithmetic operation extends values to signed Int. |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2125 |
bt = T_SHORT; |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2126 |
} |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2127 |
if (n->Opcode() == Op_LoadUB) { |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2128 |
// Adjust type for unsigned byte loads, it is important for right shifts. |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2129 |
// T_BOOLEAN is used because there is no basic type representing type |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2130 |
// TypeInt::UBYTE. Use of T_BOOLEAN for vectors is fine because only |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2131 |
// size (one byte) and sign is important. |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2132 |
bt = T_BOOLEAN; |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2133 |
} |
e376e3d428c9
8001183: incorrect results of char vectors right shift operaiton
kvn
parents:
13970
diff
changeset
|
2134 |
return Type::get_const_basic_type(bt); |
1 | 2135 |
} |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2136 |
const Type* t = _igvn.type(n); |
1 | 2137 |
if (t->basic_type() == T_INT) { |
13485
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
2138 |
// A narrow type of arithmetic operations will be determined by |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
2139 |
// propagating the type of memory operations. |
1 | 2140 |
return TypeInt::INT; |
2141 |
} |
|
2142 |
return t; |
|
2143 |
} |
|
2144 |
||
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2145 |
bool SuperWord::same_velt_type(Node* n1, Node* n2) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2146 |
const Type* vt1 = velt_type(n1); |
13885 | 2147 |
const Type* vt2 = velt_type(n2); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2148 |
if (vt1->basic_type() == T_INT && vt2->basic_type() == T_INT) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2149 |
// Compare vectors element sizes for integer types. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2150 |
return data_size(n1) == data_size(n2); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2151 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2152 |
return vt1 == vt2; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2153 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2154 |
|
1 | 2155 |
//------------------------------in_packset--------------------------- |
2156 |
// Are s1 and s2 in a pack pair and ordered as s1,s2? |
|
2157 |
bool SuperWord::in_packset(Node* s1, Node* s2) { |
|
2158 |
for (int i = 0; i < _packset.length(); i++) { |
|
2159 |
Node_List* p = _packset.at(i); |
|
2160 |
assert(p->size() == 2, "must be"); |
|
2161 |
if (p->at(0) == s1 && p->at(p->size()-1) == s2) { |
|
2162 |
return true; |
|
2163 |
} |
|
2164 |
} |
|
2165 |
return false; |
|
2166 |
} |
|
2167 |
||
2168 |
//------------------------------in_pack--------------------------- |
|
2169 |
// Is s in pack p? |
|
2170 |
Node_List* SuperWord::in_pack(Node* s, Node_List* p) { |
|
2171 |
for (uint i = 0; i < p->size(); i++) { |
|
2172 |
if (p->at(i) == s) { |
|
2173 |
return p; |
|
2174 |
} |
|
2175 |
} |
|
2176 |
return NULL; |
|
2177 |
} |
|
2178 |
||
2179 |
//------------------------------remove_pack_at--------------------------- |
|
2180 |
// Remove the pack at position pos in the packset |
|
2181 |
void SuperWord::remove_pack_at(int pos) { |
|
2182 |
Node_List* p = _packset.at(pos); |
|
2183 |
for (uint i = 0; i < p->size(); i++) { |
|
2184 |
Node* s = p->at(i); |
|
2185 |
set_my_pack(s, NULL); |
|
2186 |
} |
|
2187 |
_packset.remove_at(pos); |
|
2188 |
} |
|
2189 |
||
30211 | 2190 |
void SuperWord::packset_sort(int n) { |
2191 |
// simple bubble sort so that we capitalize with O(n) when its already sorted |
|
2192 |
while (n != 0) { |
|
2193 |
bool swapped = false; |
|
2194 |
for (int i = 1; i < n; i++) { |
|
2195 |
Node_List* q_low = _packset.at(i-1); |
|
2196 |
Node_List* q_i = _packset.at(i); |
|
2197 |
||
2198 |
// only swap when we find something to swap |
|
2199 |
if (alignment(q_low->at(0)) > alignment(q_i->at(0))) { |
|
2200 |
Node_List* t = q_i; |
|
2201 |
*(_packset.adr_at(i)) = q_low; |
|
2202 |
*(_packset.adr_at(i-1)) = q_i; |
|
2203 |
swapped = true; |
|
2204 |
} |
|
2205 |
} |
|
2206 |
if (swapped == false) break; |
|
2207 |
n--; |
|
2208 |
} |
|
2209 |
} |
|
2210 |
||
1 | 2211 |
//------------------------------executed_first--------------------------- |
2212 |
// Return the node executed first in pack p. Uses the RPO block list |
|
2213 |
// to determine order. |
|
2214 |
Node* SuperWord::executed_first(Node_List* p) { |
|
2215 |
Node* n = p->at(0); |
|
2216 |
int n_rpo = bb_idx(n); |
|
2217 |
for (uint i = 1; i < p->size(); i++) { |
|
2218 |
Node* s = p->at(i); |
|
2219 |
int s_rpo = bb_idx(s); |
|
2220 |
if (s_rpo < n_rpo) { |
|
2221 |
n = s; |
|
2222 |
n_rpo = s_rpo; |
|
2223 |
} |
|
2224 |
} |
|
2225 |
return n; |
|
2226 |
} |
|
2227 |
||
2228 |
//------------------------------executed_last--------------------------- |
|
2229 |
// Return the node executed last in pack p. |
|
2230 |
Node* SuperWord::executed_last(Node_List* p) { |
|
2231 |
Node* n = p->at(0); |
|
2232 |
int n_rpo = bb_idx(n); |
|
2233 |
for (uint i = 1; i < p->size(); i++) { |
|
2234 |
Node* s = p->at(i); |
|
2235 |
int s_rpo = bb_idx(s); |
|
2236 |
if (s_rpo > n_rpo) { |
|
2237 |
n = s; |
|
2238 |
n_rpo = s_rpo; |
|
2239 |
} |
|
2240 |
} |
|
2241 |
return n; |
|
2242 |
} |
|
2243 |
||
2244 |
//----------------------------align_initial_loop_index--------------------------- |
|
2245 |
// Adjust pre-loop limit so that in main loop, a load/store reference |
|
2246 |
// to align_to_ref will be a position zero in the vector. |
|
2247 |
// (iv + k) mod vector_align == 0 |
|
2248 |
void SuperWord::align_initial_loop_index(MemNode* align_to_ref) { |
|
2249 |
CountedLoopNode *main_head = lp()->as_CountedLoop(); |
|
2250 |
assert(main_head->is_main_loop(), ""); |
|
2251 |
CountedLoopEndNode* pre_end = get_pre_loop_end(main_head); |
|
22919
4d686766ac23
8010500: [parfait] Possible null pointer dereference at hotspot/src/share/vm/opto/loopnode.hpp
adlertz
parents:
22234
diff
changeset
|
2252 |
assert(pre_end != NULL, "we must have a correct pre-loop"); |
1 | 2253 |
Node *pre_opaq1 = pre_end->limit(); |
2254 |
assert(pre_opaq1->Opcode() == Op_Opaque1, ""); |
|
2255 |
Opaque1Node *pre_opaq = (Opaque1Node*)pre_opaq1; |
|
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2256 |
Node *lim0 = pre_opaq->in(1); |
1 | 2257 |
|
2258 |
// Where we put new limit calculations |
|
2259 |
Node *pre_ctrl = pre_end->loopnode()->in(LoopNode::EntryControl); |
|
2260 |
||
2261 |
// Ensure the original loop limit is available from the |
|
2262 |
// pre-loop Opaque1 node. |
|
2263 |
Node *orig_limit = pre_opaq->original_loop_limit(); |
|
2264 |
assert(orig_limit != NULL && _igvn.type(orig_limit) != Type::TOP, ""); |
|
2265 |
||
2266 |
SWPointer align_to_ref_p(align_to_ref, this); |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2267 |
assert(align_to_ref_p.valid(), "sanity"); |
1 | 2268 |
|
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2269 |
// Given: |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2270 |
// lim0 == original pre loop limit |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2271 |
// V == v_align (power of 2) |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2272 |
// invar == extra invariant piece of the address expression |
13485
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
2273 |
// e == offset [ +/- invar ] |
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2274 |
// |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2275 |
// When reassociating expressions involving '%' the basic rules are: |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2276 |
// (a - b) % k == 0 => a % k == b % k |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2277 |
// and: |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2278 |
// (a + b) % k == 0 => a % k == (k - b) % k |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2279 |
// |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2280 |
// For stride > 0 && scale > 0, |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2281 |
// Derive the new pre-loop limit "lim" such that the two constraints: |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2282 |
// (1) lim = lim0 + N (where N is some positive integer < V) |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2283 |
// (2) (e + lim) % V == 0 |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2284 |
// are true. |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2285 |
// |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2286 |
// Substituting (1) into (2), |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2287 |
// (e + lim0 + N) % V == 0 |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2288 |
// solve for N: |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2289 |
// N = (V - (e + lim0)) % V |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2290 |
// substitute back into (1), so that new limit |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2291 |
// lim = lim0 + (V - (e + lim0)) % V |
1 | 2292 |
// |
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2293 |
// For stride > 0 && scale < 0 |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2294 |
// Constraints: |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2295 |
// lim = lim0 + N |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2296 |
// (e - lim) % V == 0 |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2297 |
// Solving for lim: |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2298 |
// (e - lim0 - N) % V == 0 |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2299 |
// N = (e - lim0) % V |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2300 |
// lim = lim0 + (e - lim0) % V |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2301 |
// |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2302 |
// For stride < 0 && scale > 0 |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2303 |
// Constraints: |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2304 |
// lim = lim0 - N |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2305 |
// (e + lim) % V == 0 |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2306 |
// Solving for lim: |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2307 |
// (e + lim0 - N) % V == 0 |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2308 |
// N = (e + lim0) % V |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2309 |
// lim = lim0 - (e + lim0) % V |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2310 |
// |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2311 |
// For stride < 0 && scale < 0 |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2312 |
// Constraints: |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2313 |
// lim = lim0 - N |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2314 |
// (e - lim) % V == 0 |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2315 |
// Solving for lim: |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2316 |
// (e - lim0 + N) % V == 0 |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2317 |
// N = (V - (e - lim0)) % V |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2318 |
// lim = lim0 - (V - (e - lim0)) % V |
1 | 2319 |
|
13108
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
2320 |
int vw = vector_width_in_bytes(align_to_ref); |
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2321 |
int stride = iv_stride(); |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2322 |
int scale = align_to_ref_p.scale_in_bytes(); |
1 | 2323 |
int elt_size = align_to_ref_p.memory_size(); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2324 |
int v_align = vw / elt_size; |
13108
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
2325 |
assert(v_align > 1, "sanity"); |
13485
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
2326 |
int offset = align_to_ref_p.offset_in_bytes() / elt_size; |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
2327 |
Node *offsn = _igvn.intcon(offset); |
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2328 |
|
13485
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
2329 |
Node *e = offsn; |
1 | 2330 |
if (align_to_ref_p.invar() != NULL) { |
13485
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
2331 |
// incorporate any extra invariant piece producing (offset +/- invar) >>> log2(elt) |
1 | 2332 |
Node* log2_elt = _igvn.intcon(exact_log2(elt_size)); |
24923
9631f7d691dc
8034812: remove IDX_INIT macro hack in Node class
thartmann
parents:
24090
diff
changeset
|
2333 |
Node* aref = new URShiftINode(align_to_ref_p.invar(), log2_elt); |
13894 | 2334 |
_igvn.register_new_node_with_optimizer(aref); |
1 | 2335 |
_phase->set_ctrl(aref, pre_ctrl); |
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2336 |
if (align_to_ref_p.negate_invar()) { |
24923
9631f7d691dc
8034812: remove IDX_INIT macro hack in Node class
thartmann
parents:
24090
diff
changeset
|
2337 |
e = new SubINode(e, aref); |
1 | 2338 |
} else { |
24923
9631f7d691dc
8034812: remove IDX_INIT macro hack in Node class
thartmann
parents:
24090
diff
changeset
|
2339 |
e = new AddINode(e, aref); |
1 | 2340 |
} |
13894 | 2341 |
_igvn.register_new_node_with_optimizer(e); |
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2342 |
_phase->set_ctrl(e, pre_ctrl); |
1 | 2343 |
} |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2344 |
if (vw > ObjectAlignmentInBytes) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2345 |
// incorporate base e +/- base && Mask >>> log2(elt) |
24923
9631f7d691dc
8034812: remove IDX_INIT macro hack in Node class
thartmann
parents:
24090
diff
changeset
|
2346 |
Node* xbase = new CastP2XNode(NULL, align_to_ref_p.base()); |
13894 | 2347 |
_igvn.register_new_node_with_optimizer(xbase); |
13485
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
2348 |
#ifdef _LP64 |
24923
9631f7d691dc
8034812: remove IDX_INIT macro hack in Node class
thartmann
parents:
24090
diff
changeset
|
2349 |
xbase = new ConvL2INode(xbase); |
13894 | 2350 |
_igvn.register_new_node_with_optimizer(xbase); |
13485
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
2351 |
#endif |
6c7faa516fc6
6340864: Implement vectorization optimizations in hotspot-server
kvn
parents:
13108
diff
changeset
|
2352 |
Node* mask = _igvn.intcon(vw-1); |
24923
9631f7d691dc
8034812: remove IDX_INIT macro hack in Node class
thartmann
parents:
24090
diff
changeset
|
2353 |
Node* masked_xbase = new AndINode(xbase, mask); |
13894 | 2354 |
_igvn.register_new_node_with_optimizer(masked_xbase); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2355 |
Node* log2_elt = _igvn.intcon(exact_log2(elt_size)); |
24923
9631f7d691dc
8034812: remove IDX_INIT macro hack in Node class
thartmann
parents:
24090
diff
changeset
|
2356 |
Node* bref = new URShiftINode(masked_xbase, log2_elt); |
13894 | 2357 |
_igvn.register_new_node_with_optimizer(bref); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2358 |
_phase->set_ctrl(bref, pre_ctrl); |
24923
9631f7d691dc
8034812: remove IDX_INIT macro hack in Node class
thartmann
parents:
24090
diff
changeset
|
2359 |
e = new AddINode(e, bref); |
13894 | 2360 |
_igvn.register_new_node_with_optimizer(e); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2361 |
_phase->set_ctrl(e, pre_ctrl); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2362 |
} |
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2363 |
|
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2364 |
// compute e +/- lim0 |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2365 |
if (scale < 0) { |
24923
9631f7d691dc
8034812: remove IDX_INIT macro hack in Node class
thartmann
parents:
24090
diff
changeset
|
2366 |
e = new SubINode(e, lim0); |
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2367 |
} else { |
24923
9631f7d691dc
8034812: remove IDX_INIT macro hack in Node class
thartmann
parents:
24090
diff
changeset
|
2368 |
e = new AddINode(e, lim0); |
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2369 |
} |
13894 | 2370 |
_igvn.register_new_node_with_optimizer(e); |
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2371 |
_phase->set_ctrl(e, pre_ctrl); |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2372 |
|
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2373 |
if (stride * scale > 0) { |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2374 |
// compute V - (e +/- lim0) |
1 | 2375 |
Node* va = _igvn.intcon(v_align); |
24923
9631f7d691dc
8034812: remove IDX_INIT macro hack in Node class
thartmann
parents:
24090
diff
changeset
|
2376 |
e = new SubINode(va, e); |
13894 | 2377 |
_igvn.register_new_node_with_optimizer(e); |
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2378 |
_phase->set_ctrl(e, pre_ctrl); |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2379 |
} |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2380 |
// compute N = (exp) % V |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2381 |
Node* va_msk = _igvn.intcon(v_align - 1); |
24923
9631f7d691dc
8034812: remove IDX_INIT macro hack in Node class
thartmann
parents:
24090
diff
changeset
|
2382 |
Node* N = new AndINode(e, va_msk); |
13894 | 2383 |
_igvn.register_new_node_with_optimizer(N); |
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2384 |
_phase->set_ctrl(N, pre_ctrl); |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2385 |
|
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2386 |
// substitute back into (1), so that new limit |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2387 |
// lim = lim0 + N |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2388 |
Node* lim; |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2389 |
if (stride < 0) { |
24923
9631f7d691dc
8034812: remove IDX_INIT macro hack in Node class
thartmann
parents:
24090
diff
changeset
|
2390 |
lim = new SubINode(lim0, N); |
1 | 2391 |
} else { |
24923
9631f7d691dc
8034812: remove IDX_INIT macro hack in Node class
thartmann
parents:
24090
diff
changeset
|
2392 |
lim = new AddINode(lim0, N); |
1 | 2393 |
} |
13894 | 2394 |
_igvn.register_new_node_with_optimizer(lim); |
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2395 |
_phase->set_ctrl(lim, pre_ctrl); |
1 | 2396 |
Node* constrained = |
24923
9631f7d691dc
8034812: remove IDX_INIT macro hack in Node class
thartmann
parents:
24090
diff
changeset
|
2397 |
(stride > 0) ? (Node*) new MinINode(lim, orig_limit) |
9631f7d691dc
8034812: remove IDX_INIT macro hack in Node class
thartmann
parents:
24090
diff
changeset
|
2398 |
: (Node*) new MaxINode(lim, orig_limit); |
13894 | 2399 |
_igvn.register_new_node_with_optimizer(constrained); |
1 | 2400 |
_phase->set_ctrl(constrained, pre_ctrl); |
2401 |
_igvn.hash_delete(pre_opaq); |
|
2402 |
pre_opaq->set_req(1, constrained); |
|
2403 |
} |
|
2404 |
||
2405 |
//----------------------------get_pre_loop_end--------------------------- |
|
2406 |
// Find pre loop end from main loop. Returns null if none. |
|
2407 |
CountedLoopEndNode* SuperWord::get_pre_loop_end(CountedLoopNode *cl) { |
|
2408 |
Node *ctrl = cl->in(LoopNode::EntryControl); |
|
2409 |
if (!ctrl->is_IfTrue() && !ctrl->is_IfFalse()) return NULL; |
|
2410 |
Node *iffm = ctrl->in(0); |
|
2411 |
if (!iffm->is_If()) return NULL; |
|
2412 |
Node *p_f = iffm->in(0); |
|
2413 |
if (!p_f->is_IfFalse()) return NULL; |
|
2414 |
if (!p_f->in(0)->is_CountedLoopEnd()) return NULL; |
|
2415 |
CountedLoopEndNode *pre_end = p_f->in(0)->as_CountedLoopEnd(); |
|
22919
4d686766ac23
8010500: [parfait] Possible null pointer dereference at hotspot/src/share/vm/opto/loopnode.hpp
adlertz
parents:
22234
diff
changeset
|
2416 |
CountedLoopNode* loop_node = pre_end->loopnode(); |
4d686766ac23
8010500: [parfait] Possible null pointer dereference at hotspot/src/share/vm/opto/loopnode.hpp
adlertz
parents:
22234
diff
changeset
|
2417 |
if (loop_node == NULL || !loop_node->is_pre_loop()) return NULL; |
1 | 2418 |
return pre_end; |
2419 |
} |
|
2420 |
||
2421 |
||
2422 |
//------------------------------init--------------------------- |
|
2423 |
void SuperWord::init() { |
|
2424 |
_dg.init(); |
|
2425 |
_packset.clear(); |
|
2426 |
_disjoint_ptrs.clear(); |
|
2427 |
_block.clear(); |
|
2428 |
_data_entry.clear(); |
|
2429 |
_mem_slice_head.clear(); |
|
2430 |
_mem_slice_tail.clear(); |
|
30593 | 2431 |
_iteration_first.clear(); |
2432 |
_iteration_last.clear(); |
|
1 | 2433 |
_node_info.clear(); |
2434 |
_align_to_ref = NULL; |
|
2435 |
_lpt = NULL; |
|
2436 |
_lp = NULL; |
|
2437 |
_bb = NULL; |
|
2438 |
_iv = NULL; |
|
30588 | 2439 |
_race_possible = 0; |
2440 |
_num_work_vecs = 0; |
|
2441 |
_num_reductions = 0; |
|
1 | 2442 |
} |
2443 |
||
30593 | 2444 |
//------------------------------restart--------------------------- |
2445 |
void SuperWord::restart() { |
|
2446 |
_dg.init(); |
|
2447 |
_packset.clear(); |
|
2448 |
_disjoint_ptrs.clear(); |
|
2449 |
_block.clear(); |
|
2450 |
_data_entry.clear(); |
|
2451 |
_mem_slice_head.clear(); |
|
2452 |
_mem_slice_tail.clear(); |
|
2453 |
_node_info.clear(); |
|
2454 |
} |
|
2455 |
||
1 | 2456 |
//------------------------------print_packset--------------------------- |
2457 |
void SuperWord::print_packset() { |
|
2458 |
#ifndef PRODUCT |
|
2459 |
tty->print_cr("packset"); |
|
2460 |
for (int i = 0; i < _packset.length(); i++) { |
|
2461 |
tty->print_cr("Pack: %d", i); |
|
2462 |
Node_List* p = _packset.at(i); |
|
2463 |
print_pack(p); |
|
2464 |
} |
|
2465 |
#endif |
|
2466 |
} |
|
2467 |
||
2468 |
//------------------------------print_pack--------------------------- |
|
2469 |
void SuperWord::print_pack(Node_List* p) { |
|
2470 |
for (uint i = 0; i < p->size(); i++) { |
|
2471 |
print_stmt(p->at(i)); |
|
2472 |
} |
|
2473 |
} |
|
2474 |
||
2475 |
//------------------------------print_bb--------------------------- |
|
2476 |
void SuperWord::print_bb() { |
|
2477 |
#ifndef PRODUCT |
|
2478 |
tty->print_cr("\nBlock"); |
|
2479 |
for (int i = 0; i < _block.length(); i++) { |
|
2480 |
Node* n = _block.at(i); |
|
2481 |
tty->print("%d ", i); |
|
2482 |
if (n) { |
|
2483 |
n->dump(); |
|
2484 |
} |
|
2485 |
} |
|
2486 |
#endif |
|
2487 |
} |
|
2488 |
||
2489 |
//------------------------------print_stmt--------------------------- |
|
2490 |
void SuperWord::print_stmt(Node* s) { |
|
2491 |
#ifndef PRODUCT |
|
2492 |
tty->print(" align: %d \t", alignment(s)); |
|
2493 |
s->dump(); |
|
2494 |
#endif |
|
2495 |
} |
|
2496 |
||
2497 |
//------------------------------blank--------------------------- |
|
2498 |
char* SuperWord::blank(uint depth) { |
|
2499 |
static char blanks[101]; |
|
2500 |
assert(depth < 101, "too deep"); |
|
2501 |
for (uint i = 0; i < depth; i++) blanks[i] = ' '; |
|
2502 |
blanks[depth] = '\0'; |
|
2503 |
return blanks; |
|
2504 |
} |
|
2505 |
||
2506 |
||
2507 |
//==============================SWPointer=========================== |
|
2508 |
||
2509 |
//----------------------------SWPointer------------------------ |
|
2510 |
SWPointer::SWPointer(MemNode* mem, SuperWord* slp) : |
|
2511 |
_mem(mem), _slp(slp), _base(NULL), _adr(NULL), |
|
2512 |
_scale(0), _offset(0), _invar(NULL), _negate_invar(false) { |
|
2513 |
||
2514 |
Node* adr = mem->in(MemNode::Address); |
|
2515 |
if (!adr->is_AddP()) { |
|
2516 |
assert(!valid(), "too complex"); |
|
2517 |
return; |
|
2518 |
} |
|
2519 |
// Match AddP(base, AddP(ptr, k*iv [+ invariant]), constant) |
|
2520 |
Node* base = adr->in(AddPNode::Base); |
|
4428
d1617f46285d
6852078: HSX 14/16 in jdk 5.0: api/javax_management api/org_omg jck tests crashes or make tnameserv crash
cfang
parents:
3906
diff
changeset
|
2521 |
//unsafe reference could not be aligned appropriately without runtime checking |
d1617f46285d
6852078: HSX 14/16 in jdk 5.0: api/javax_management api/org_omg jck tests crashes or make tnameserv crash
cfang
parents:
3906
diff
changeset
|
2522 |
if (base == NULL || base->bottom_type() == Type::TOP) { |
d1617f46285d
6852078: HSX 14/16 in jdk 5.0: api/javax_management api/org_omg jck tests crashes or make tnameserv crash
cfang
parents:
3906
diff
changeset
|
2523 |
assert(!valid(), "unsafe access"); |
d1617f46285d
6852078: HSX 14/16 in jdk 5.0: api/javax_management api/org_omg jck tests crashes or make tnameserv crash
cfang
parents:
3906
diff
changeset
|
2524 |
return; |
d1617f46285d
6852078: HSX 14/16 in jdk 5.0: api/javax_management api/org_omg jck tests crashes or make tnameserv crash
cfang
parents:
3906
diff
changeset
|
2525 |
} |
1 | 2526 |
for (int i = 0; i < 3; i++) { |
2527 |
if (!scaled_iv_plus_offset(adr->in(AddPNode::Offset))) { |
|
2528 |
assert(!valid(), "too complex"); |
|
2529 |
return; |
|
2530 |
} |
|
2531 |
adr = adr->in(AddPNode::Address); |
|
2532 |
if (base == adr || !adr->is_AddP()) { |
|
2533 |
break; // stop looking at addp's |
|
2534 |
} |
|
2535 |
} |
|
2536 |
_base = base; |
|
2537 |
_adr = adr; |
|
2538 |
assert(valid(), "Usable"); |
|
2539 |
} |
|
2540 |
||
2541 |
// Following is used to create a temporary object during |
|
2542 |
// the pattern match of an address expression. |
|
2543 |
SWPointer::SWPointer(SWPointer* p) : |
|
2544 |
_mem(p->_mem), _slp(p->_slp), _base(NULL), _adr(NULL), |
|
2545 |
_scale(0), _offset(0), _invar(NULL), _negate_invar(false) {} |
|
2546 |
||
2547 |
//------------------------scaled_iv_plus_offset-------------------- |
|
2548 |
// Match: k*iv + offset |
|
2549 |
// where: k is a constant that maybe zero, and |
|
2550 |
// offset is (k2 [+/- invariant]) where k2 maybe zero and invariant is optional |
|
2551 |
bool SWPointer::scaled_iv_plus_offset(Node* n) { |
|
2552 |
if (scaled_iv(n)) { |
|
2553 |
return true; |
|
2554 |
} |
|
2555 |
if (offset_plus_k(n)) { |
|
2556 |
return true; |
|
2557 |
} |
|
2558 |
int opc = n->Opcode(); |
|
2559 |
if (opc == Op_AddI) { |
|
2560 |
if (scaled_iv(n->in(1)) && offset_plus_k(n->in(2))) { |
|
2561 |
return true; |
|
2562 |
} |
|
2563 |
if (scaled_iv(n->in(2)) && offset_plus_k(n->in(1))) { |
|
2564 |
return true; |
|
2565 |
} |
|
2566 |
} else if (opc == Op_SubI) { |
|
2567 |
if (scaled_iv(n->in(1)) && offset_plus_k(n->in(2), true)) { |
|
2568 |
return true; |
|
2569 |
} |
|
2570 |
if (scaled_iv(n->in(2)) && offset_plus_k(n->in(1))) { |
|
2571 |
_scale *= -1; |
|
2572 |
return true; |
|
2573 |
} |
|
2574 |
} |
|
2575 |
return false; |
|
2576 |
} |
|
2577 |
||
2578 |
//----------------------------scaled_iv------------------------ |
|
2579 |
// Match: k*iv where k is a constant that's not zero |
|
2580 |
bool SWPointer::scaled_iv(Node* n) { |
|
2581 |
if (_scale != 0) { |
|
2582 |
return false; // already found a scale |
|
2583 |
} |
|
2584 |
if (n == iv()) { |
|
2585 |
_scale = 1; |
|
2586 |
return true; |
|
2587 |
} |
|
2588 |
int opc = n->Opcode(); |
|
2589 |
if (opc == Op_MulI) { |
|
2590 |
if (n->in(1) == iv() && n->in(2)->is_Con()) { |
|
2591 |
_scale = n->in(2)->get_int(); |
|
2592 |
return true; |
|
2593 |
} else if (n->in(2) == iv() && n->in(1)->is_Con()) { |
|
2594 |
_scale = n->in(1)->get_int(); |
|
2595 |
return true; |
|
2596 |
} |
|
2597 |
} else if (opc == Op_LShiftI) { |
|
2598 |
if (n->in(1) == iv() && n->in(2)->is_Con()) { |
|
2599 |
_scale = 1 << n->in(2)->get_int(); |
|
2600 |
return true; |
|
2601 |
} |
|
2602 |
} else if (opc == Op_ConvI2L) { |
|
2603 |
if (scaled_iv_plus_offset(n->in(1))) { |
|
2604 |
return true; |
|
2605 |
} |
|
2606 |
} else if (opc == Op_LShiftL) { |
|
2607 |
if (!has_iv() && _invar == NULL) { |
|
2608 |
// Need to preserve the current _offset value, so |
|
2609 |
// create a temporary object for this expression subtree. |
|
2610 |
// Hacky, so should re-engineer the address pattern match. |
|
2611 |
SWPointer tmp(this); |
|
2612 |
if (tmp.scaled_iv_plus_offset(n->in(1))) { |
|
2613 |
if (tmp._invar == NULL) { |
|
2614 |
int mult = 1 << n->in(2)->get_int(); |
|
2615 |
_scale = tmp._scale * mult; |
|
2616 |
_offset += tmp._offset * mult; |
|
2617 |
return true; |
|
2618 |
} |
|
2619 |
} |
|
2620 |
} |
|
2621 |
} |
|
2622 |
return false; |
|
2623 |
} |
|
2624 |
||
2625 |
//----------------------------offset_plus_k------------------------ |
|
2626 |
// Match: offset is (k [+/- invariant]) |
|
2627 |
// where k maybe zero and invariant is optional, but not both. |
|
2628 |
bool SWPointer::offset_plus_k(Node* n, bool negate) { |
|
2629 |
int opc = n->Opcode(); |
|
2630 |
if (opc == Op_ConI) { |
|
2631 |
_offset += negate ? -(n->get_int()) : n->get_int(); |
|
2632 |
return true; |
|
2633 |
} else if (opc == Op_ConL) { |
|
2634 |
// Okay if value fits into an int |
|
2635 |
const TypeLong* t = n->find_long_type(); |
|
2636 |
if (t->higher_equal(TypeLong::INT)) { |
|
2637 |
jlong loff = n->get_long(); |
|
2638 |
jint off = (jint)loff; |
|
2639 |
_offset += negate ? -off : loff; |
|
2640 |
return true; |
|
2641 |
} |
|
2642 |
return false; |
|
2643 |
} |
|
2644 |
if (_invar != NULL) return false; // already have an invariant |
|
2645 |
if (opc == Op_AddI) { |
|
2646 |
if (n->in(2)->is_Con() && invariant(n->in(1))) { |
|
2647 |
_negate_invar = negate; |
|
2648 |
_invar = n->in(1); |
|
2649 |
_offset += negate ? -(n->in(2)->get_int()) : n->in(2)->get_int(); |
|
2650 |
return true; |
|
2651 |
} else if (n->in(1)->is_Con() && invariant(n->in(2))) { |
|
2652 |
_offset += negate ? -(n->in(1)->get_int()) : n->in(1)->get_int(); |
|
2653 |
_negate_invar = negate; |
|
2654 |
_invar = n->in(2); |
|
2655 |
return true; |
|
2656 |
} |
|
2657 |
} |
|
2658 |
if (opc == Op_SubI) { |
|
2659 |
if (n->in(2)->is_Con() && invariant(n->in(1))) { |
|
2660 |
_negate_invar = negate; |
|
2661 |
_invar = n->in(1); |
|
2662 |
_offset += !negate ? -(n->in(2)->get_int()) : n->in(2)->get_int(); |
|
2663 |
return true; |
|
2664 |
} else if (n->in(1)->is_Con() && invariant(n->in(2))) { |
|
2665 |
_offset += negate ? -(n->in(1)->get_int()) : n->in(1)->get_int(); |
|
2666 |
_negate_invar = !negate; |
|
2667 |
_invar = n->in(2); |
|
2668 |
return true; |
|
2669 |
} |
|
2670 |
} |
|
2671 |
if (invariant(n)) { |
|
2672 |
_negate_invar = negate; |
|
2673 |
_invar = n; |
|
2674 |
return true; |
|
2675 |
} |
|
2676 |
return false; |
|
2677 |
} |
|
2678 |
||
2679 |
//----------------------------print------------------------ |
|
2680 |
void SWPointer::print() { |
|
2681 |
#ifndef PRODUCT |
|
2682 |
tty->print("base: %d adr: %d scale: %d offset: %d invar: %c%d\n", |
|
2683 |
_base != NULL ? _base->_idx : 0, |
|
2684 |
_adr != NULL ? _adr->_idx : 0, |
|
2685 |
_scale, _offset, |
|
2686 |
_negate_invar?'-':'+', |
|
2687 |
_invar != NULL ? _invar->_idx : 0); |
|
2688 |
#endif |
|
2689 |
} |
|
2690 |
||
2691 |
// ========================= OrderedPair ===================== |
|
2692 |
||
2693 |
const OrderedPair OrderedPair::initial; |
|
2694 |
||
2695 |
// ========================= SWNodeInfo ===================== |
|
2696 |
||
2697 |
const SWNodeInfo SWNodeInfo::initial; |
|
2698 |
||
2699 |
||
2700 |
// ============================ DepGraph =========================== |
|
2701 |
||
2702 |
//------------------------------make_node--------------------------- |
|
2703 |
// Make a new dependence graph node for an ideal node. |
|
2704 |
DepMem* DepGraph::make_node(Node* node) { |
|
2705 |
DepMem* m = new (_arena) DepMem(node); |
|
2706 |
if (node != NULL) { |
|
2707 |
assert(_map.at_grow(node->_idx) == NULL, "one init only"); |
|
2708 |
_map.at_put_grow(node->_idx, m); |
|
2709 |
} |
|
2710 |
return m; |
|
2711 |
} |
|
2712 |
||
2713 |
//------------------------------make_edge--------------------------- |
|
2714 |
// Make a new dependence graph edge from dpred -> dsucc |
|
2715 |
DepEdge* DepGraph::make_edge(DepMem* dpred, DepMem* dsucc) { |
|
2716 |
DepEdge* e = new (_arena) DepEdge(dpred, dsucc, dsucc->in_head(), dpred->out_head()); |
|
2717 |
dpred->set_out_head(e); |
|
2718 |
dsucc->set_in_head(e); |
|
2719 |
return e; |
|
2720 |
} |
|
2721 |
||
2722 |
// ========================== DepMem ======================== |
|
2723 |
||
2724 |
//------------------------------in_cnt--------------------------- |
|
2725 |
int DepMem::in_cnt() { |
|
2726 |
int ct = 0; |
|
2727 |
for (DepEdge* e = _in_head; e != NULL; e = e->next_in()) ct++; |
|
2728 |
return ct; |
|
2729 |
} |
|
2730 |
||
2731 |
//------------------------------out_cnt--------------------------- |
|
2732 |
int DepMem::out_cnt() { |
|
2733 |
int ct = 0; |
|
2734 |
for (DepEdge* e = _out_head; e != NULL; e = e->next_out()) ct++; |
|
2735 |
return ct; |
|
2736 |
} |
|
2737 |
||
2738 |
//------------------------------print----------------------------- |
|
2739 |
void DepMem::print() { |
|
2740 |
#ifndef PRODUCT |
|
2741 |
tty->print(" DepNode %d (", _node->_idx); |
|
2742 |
for (DepEdge* p = _in_head; p != NULL; p = p->next_in()) { |
|
2743 |
Node* pred = p->pred()->node(); |
|
2744 |
tty->print(" %d", pred != NULL ? pred->_idx : 0); |
|
2745 |
} |
|
2746 |
tty->print(") ["); |
|
2747 |
for (DepEdge* s = _out_head; s != NULL; s = s->next_out()) { |
|
2748 |
Node* succ = s->succ()->node(); |
|
2749 |
tty->print(" %d", succ != NULL ? succ->_idx : 0); |
|
2750 |
} |
|
2751 |
tty->print_cr(" ]"); |
|
2752 |
#endif |
|
2753 |
} |
|
2754 |
||
2755 |
// =========================== DepEdge ========================= |
|
2756 |
||
2757 |
//------------------------------DepPreds--------------------------- |
|
2758 |
void DepEdge::print() { |
|
2759 |
#ifndef PRODUCT |
|
2760 |
tty->print_cr("DepEdge: %d [ %d ]", _pred->node()->_idx, _succ->node()->_idx); |
|
2761 |
#endif |
|
2762 |
} |
|
2763 |
||
2764 |
// =========================== DepPreds ========================= |
|
2765 |
// Iterator over predecessor edges in the dependence graph. |
|
2766 |
||
2767 |
//------------------------------DepPreds--------------------------- |
|
2768 |
DepPreds::DepPreds(Node* n, DepGraph& dg) { |
|
2769 |
_n = n; |
|
2770 |
_done = false; |
|
2771 |
if (_n->is_Store() || _n->is_Load()) { |
|
2772 |
_next_idx = MemNode::Address; |
|
2773 |
_end_idx = n->req(); |
|
2774 |
_dep_next = dg.dep(_n)->in_head(); |
|
2775 |
} else if (_n->is_Mem()) { |
|
2776 |
_next_idx = 0; |
|
2777 |
_end_idx = 0; |
|
2778 |
_dep_next = dg.dep(_n)->in_head(); |
|
2779 |
} else { |
|
2780 |
_next_idx = 1; |
|
2781 |
_end_idx = _n->req(); |
|
2782 |
_dep_next = NULL; |
|
2783 |
} |
|
2784 |
next(); |
|
2785 |
} |
|
2786 |
||
2787 |
//------------------------------next--------------------------- |
|
2788 |
void DepPreds::next() { |
|
2789 |
if (_dep_next != NULL) { |
|
2790 |
_current = _dep_next->pred()->node(); |
|
2791 |
_dep_next = _dep_next->next_in(); |
|
2792 |
} else if (_next_idx < _end_idx) { |
|
2793 |
_current = _n->in(_next_idx++); |
|
2794 |
} else { |
|
2795 |
_done = true; |
|
2796 |
} |
|
2797 |
} |
|
2798 |
||
2799 |
// =========================== DepSuccs ========================= |
|
2800 |
// Iterator over successor edges in the dependence graph. |
|
2801 |
||
2802 |
//------------------------------DepSuccs--------------------------- |
|
2803 |
DepSuccs::DepSuccs(Node* n, DepGraph& dg) { |
|
2804 |
_n = n; |
|
2805 |
_done = false; |
|
2806 |
if (_n->is_Load()) { |
|
2807 |
_next_idx = 0; |
|
2808 |
_end_idx = _n->outcnt(); |
|
2809 |
_dep_next = dg.dep(_n)->out_head(); |
|
2810 |
} else if (_n->is_Mem() || _n->is_Phi() && _n->bottom_type() == Type::MEMORY) { |
|
2811 |
_next_idx = 0; |
|
2812 |
_end_idx = 0; |
|
2813 |
_dep_next = dg.dep(_n)->out_head(); |
|
2814 |
} else { |
|
2815 |
_next_idx = 0; |
|
2816 |
_end_idx = _n->outcnt(); |
|
2817 |
_dep_next = NULL; |
|
2818 |
} |
|
2819 |
next(); |
|
2820 |
} |
|
2821 |
||
2822 |
//-------------------------------next--------------------------- |
|
2823 |
void DepSuccs::next() { |
|
2824 |
if (_dep_next != NULL) { |
|
2825 |
_current = _dep_next->succ()->node(); |
|
2826 |
_dep_next = _dep_next->next_out(); |
|
2827 |
} else if (_next_idx < _end_idx) { |
|
2828 |
_current = _n->raw_out(_next_idx++); |
|
2829 |
} else { |
|
2830 |
_done = true; |
|
2831 |
} |
|
2832 |
} |
|
30593 | 2833 |
|
2834 |
// |
|
2835 |
// --------------------------------- vectorization/simd ----------------------------------- |
|
2836 |
// |
|
2837 |
Node* SuperWord::find_phi_for_mem_dep(LoadNode* ld) { |
|
2838 |
assert(in_bb(ld), "must be in block"); |
|
2839 |
if (_clone_map.gen(ld->_idx) == _ii_first) { |
|
2840 |
#ifndef PRODUCT |
|
2841 |
if (_vector_loop_debug) { |
|
2842 |
tty->print_cr("SuperWord::find_phi_for_mem_dep _clone_map.gen(ld->_idx)=%d", |
|
2843 |
_clone_map.gen(ld->_idx)); |
|
2844 |
} |
|
2845 |
#endif |
|
2846 |
return NULL; //we think that any ld in the first gen being vectorizable |
|
2847 |
} |
|
2848 |
||
2849 |
Node* mem = ld->in(MemNode::Memory); |
|
2850 |
if (mem->outcnt() <= 1) { |
|
2851 |
// we don't want to remove the only edge from mem node to load |
|
2852 |
#ifndef PRODUCT |
|
2853 |
if (_vector_loop_debug) { |
|
2854 |
tty->print_cr("SuperWord::find_phi_for_mem_dep input node %d to load %d has no other outputs and edge mem->load cannot be removed", |
|
2855 |
mem->_idx, ld->_idx); |
|
2856 |
ld->dump(); |
|
2857 |
mem->dump(); |
|
2858 |
} |
|
2859 |
#endif |
|
2860 |
return NULL; |
|
2861 |
} |
|
2862 |
if (!in_bb(mem) || _clone_map.gen(mem->_idx) == _clone_map.gen(ld->_idx)) { |
|
2863 |
#ifndef PRODUCT |
|
2864 |
if (_vector_loop_debug) { |
|
2865 |
tty->print_cr("SuperWord::find_phi_for_mem_dep _clone_map.gen(mem->_idx)=%d", |
|
2866 |
_clone_map.gen(mem->_idx)); |
|
2867 |
} |
|
2868 |
#endif |
|
2869 |
return NULL; // does not depend on loop volatile node or depends on the same generation |
|
2870 |
} |
|
2871 |
||
2872 |
//otherwise first node should depend on mem-phi |
|
2873 |
Node* first = first_node(ld); |
|
2874 |
assert(first->is_Load(), "must be Load"); |
|
2875 |
Node* phi = first->as_Load()->in(MemNode::Memory); |
|
2876 |
if (!phi->is_Phi() || phi->bottom_type() != Type::MEMORY) { |
|
2877 |
#ifndef PRODUCT |
|
2878 |
if (_vector_loop_debug) { |
|
2879 |
tty->print_cr("SuperWord::find_phi_for_mem_dep load is not vectorizable node, since it's `first` does not take input from mem phi"); |
|
2880 |
ld->dump(); |
|
2881 |
first->dump(); |
|
2882 |
} |
|
2883 |
#endif |
|
2884 |
return NULL; |
|
2885 |
} |
|
2886 |
||
2887 |
Node* tail = 0; |
|
2888 |
for (int m = 0; m < _mem_slice_head.length(); m++) { |
|
2889 |
if (_mem_slice_head.at(m) == phi) { |
|
2890 |
tail = _mem_slice_tail.at(m); |
|
2891 |
} |
|
2892 |
} |
|
2893 |
if (tail == 0) { //test that found phi is in the list _mem_slice_head |
|
2894 |
#ifndef PRODUCT |
|
2895 |
if (_vector_loop_debug) { |
|
2896 |
tty->print_cr("SuperWord::find_phi_for_mem_dep load %d is not vectorizable node, its phi %d is not _mem_slice_head", |
|
2897 |
ld->_idx, phi->_idx); |
|
2898 |
ld->dump(); |
|
2899 |
phi->dump(); |
|
2900 |
} |
|
2901 |
#endif |
|
2902 |
return NULL; |
|
2903 |
} |
|
2904 |
||
2905 |
// now all conditions are met |
|
2906 |
return phi; |
|
2907 |
} |
|
2908 |
||
2909 |
Node* SuperWord::first_node(Node* nd) { |
|
2910 |
for (int ii = 0; ii < _iteration_first.length(); ii++) { |
|
2911 |
Node* nnn = _iteration_first.at(ii); |
|
2912 |
if (_clone_map.idx(nnn->_idx) == _clone_map.idx(nd->_idx)) { |
|
2913 |
#ifndef PRODUCT |
|
2914 |
if (_vector_loop_debug) { |
|
2915 |
tty->print_cr("SuperWord::first_node: %d is the first iteration node for %d (_clone_map.idx(nnn->_idx) = %d)", |
|
2916 |
nnn->_idx, nd->_idx, _clone_map.idx(nnn->_idx)); |
|
2917 |
} |
|
2918 |
#endif |
|
2919 |
return nnn; |
|
2920 |
} |
|
2921 |
} |
|
2922 |
||
2923 |
#ifndef PRODUCT |
|
2924 |
if (_vector_loop_debug) { |
|
2925 |
tty->print_cr("SuperWord::first_node: did not find first iteration node for %d (_clone_map.idx(nd->_idx)=%d)", |
|
2926 |
nd->_idx, _clone_map.idx(nd->_idx)); |
|
2927 |
} |
|
2928 |
#endif |
|
2929 |
return 0; |
|
2930 |
} |
|
2931 |
||
2932 |
Node* SuperWord::last_node(Node* nd) { |
|
2933 |
for (int ii = 0; ii < _iteration_last.length(); ii++) { |
|
2934 |
Node* nnn = _iteration_last.at(ii); |
|
2935 |
if (_clone_map.idx(nnn->_idx) == _clone_map.idx(nd->_idx)) { |
|
2936 |
#ifndef PRODUCT |
|
2937 |
if (_vector_loop_debug) { |
|
2938 |
tty->print_cr("SuperWord::last_node _clone_map.idx(nnn->_idx)=%d, _clone_map.idx(nd->_idx)=%d", |
|
2939 |
_clone_map.idx(nnn->_idx), _clone_map.idx(nd->_idx)); |
|
2940 |
} |
|
2941 |
#endif |
|
2942 |
return nnn; |
|
2943 |
} |
|
2944 |
} |
|
2945 |
return 0; |
|
2946 |
} |
|
2947 |
||
2948 |
int SuperWord::mark_generations() { |
|
2949 |
Node *ii_err = 0, *tail_err; |
|
2950 |
for (int i = 0; i < _mem_slice_head.length(); i++) { |
|
2951 |
Node* phi = _mem_slice_head.at(i); |
|
2952 |
assert(phi->is_Phi(), "must be phi"); |
|
2953 |
||
2954 |
Node* tail = _mem_slice_tail.at(i); |
|
2955 |
if (_ii_last == -1) { |
|
2956 |
tail_err = tail; |
|
2957 |
_ii_last = _clone_map.gen(tail->_idx); |
|
2958 |
} |
|
2959 |
else if (_ii_last != _clone_map.gen(tail->_idx)) { |
|
2960 |
#ifndef PRODUCT |
|
2961 |
if (TraceSuperWord && Verbose) { |
|
2962 |
tty->print_cr("SuperWord::mark_generations _ii_last error - found different generations in two tail nodes "); |
|
2963 |
tail->dump(); |
|
2964 |
tail_err->dump(); |
|
2965 |
} |
|
2966 |
#endif |
|
2967 |
return -1; |
|
2968 |
} |
|
2969 |
||
2970 |
// find first iteration in the loop |
|
2971 |
for (DUIterator_Fast imax, i = phi->fast_outs(imax); i < imax; i++) { |
|
2972 |
Node* ii = phi->fast_out(i); |
|
2973 |
if (in_bb(ii) && ii->is_Store()) { // we speculate that normally Stores of one and one only generation have deps from mem phi |
|
2974 |
if (_ii_first == -1) { |
|
2975 |
ii_err = ii; |
|
2976 |
_ii_first = _clone_map.gen(ii->_idx); |
|
2977 |
} else if (_ii_first != _clone_map.gen(ii->_idx)) { |
|
2978 |
#ifndef PRODUCT |
|
2979 |
if (TraceSuperWord && Verbose) { |
|
2980 |
tty->print_cr("SuperWord::mark_generations _ii_first error - found different generations in two nodes "); |
|
2981 |
ii->dump(); |
|
2982 |
ii_err->dump(); |
|
2983 |
} |
|
2984 |
#endif |
|
2985 |
return -1; // this phi has Stores from different generations of unroll and cannot be simd/vectorized |
|
2986 |
} |
|
2987 |
} |
|
2988 |
}//for (DUIterator_Fast imax, |
|
2989 |
}//for (int i... |
|
2990 |
||
2991 |
if (_ii_first == -1 || _ii_last == -1) { |
|
2992 |
#ifndef PRODUCT |
|
2993 |
if (TraceSuperWord && Verbose) { |
|
2994 |
tty->print_cr("SuperWord::mark_generations unknown error, something vent wrong"); |
|
2995 |
} |
|
2996 |
#endif |
|
2997 |
return -1; // something vent wrong |
|
2998 |
} |
|
2999 |
// collect nodes in the first and last generations |
|
3000 |
assert(_iteration_first.length() == 0, "_iteration_first must be empty"); |
|
3001 |
assert(_iteration_last.length() == 0, "_iteration_last must be empty"); |
|
3002 |
for (int j = 0; j < _block.length(); j++) { |
|
3003 |
Node* n = _block.at(j); |
|
3004 |
node_idx_t gen = _clone_map.gen(n->_idx); |
|
3005 |
if ((signed)gen == _ii_first) { |
|
3006 |
_iteration_first.push(n); |
|
3007 |
} else if ((signed)gen == _ii_last) { |
|
3008 |
_iteration_last.push(n); |
|
3009 |
} |
|
3010 |
} |
|
3011 |
||
3012 |
// building order of iterations |
|
3013 |
assert(_ii_order.length() == 0, "should be empty"); |
|
3014 |
if (ii_err != 0) { |
|
3015 |
assert(in_bb(ii_err) && ii_err->is_Store(), "should be Store in bb"); |
|
3016 |
Node* nd = ii_err; |
|
3017 |
while(_clone_map.gen(nd->_idx) != _ii_last) { |
|
3018 |
_ii_order.push(_clone_map.gen(nd->_idx)); |
|
3019 |
bool found = false; |
|
3020 |
for (DUIterator_Fast imax, i = nd->fast_outs(imax); i < imax; i++) { |
|
3021 |
Node* use = nd->fast_out(i); |
|
3022 |
if (_clone_map.idx(use->_idx) == _clone_map.idx(nd->_idx) && use->as_Store()->in(MemNode::Memory) == nd) { |
|
3023 |
found = true; |
|
3024 |
nd = use; |
|
3025 |
break; |
|
3026 |
} |
|
3027 |
}//for |
|
3028 |
||
3029 |
if (found == false) { |
|
3030 |
#ifndef PRODUCT |
|
3031 |
if (TraceSuperWord && Verbose) { |
|
3032 |
tty->print_cr("SuperWord::mark_generations: Cannot build order of iterations - no dependent Store for %d", nd->_idx); |
|
3033 |
} |
|
3034 |
#endif |
|
3035 |
_ii_order.clear(); |
|
3036 |
return -1; |
|
3037 |
} |
|
3038 |
} //while |
|
3039 |
_ii_order.push(_clone_map.gen(nd->_idx)); |
|
3040 |
} |
|
3041 |
||
3042 |
#ifndef PRODUCT |
|
3043 |
if (_vector_loop_debug) { |
|
3044 |
tty->print_cr("SuperWord::mark_generations"); |
|
3045 |
tty->print_cr("First generation (%d) nodes:", _ii_first); |
|
3046 |
for (int ii = 0; ii < _iteration_first.length(); ii++) _iteration_first.at(ii)->dump(); |
|
3047 |
tty->print_cr("Last generation (%d) nodes:", _ii_last); |
|
3048 |
for (int ii = 0; ii < _iteration_last.length(); ii++) _iteration_last.at(ii)->dump(); |
|
3049 |
tty->print_cr(" "); |
|
3050 |
||
3051 |
tty->print("SuperWord::List of generations: "); |
|
3052 |
for (int jj = 0; jj < _ii_order.length(); ++jj) { |
|
3053 |
tty->print("%d:%d ", jj, _ii_order.at(jj)); |
|
3054 |
} |
|
3055 |
tty->print_cr(" "); |
|
3056 |
} |
|
3057 |
#endif |
|
3058 |
||
3059 |
return _ii_first; |
|
3060 |
} |
|
3061 |
||
3062 |
bool SuperWord::fix_commutative_inputs(Node* gold, Node* fix) { |
|
3063 |
assert(gold->is_Add() && fix->is_Add() || gold->is_Mul() && fix->is_Mul(), "should be only Add or Mul nodes"); |
|
3064 |
assert(_clone_map.idx(gold->_idx) == _clone_map.idx(fix->_idx), "should be clones of the same node"); |
|
3065 |
Node* gin1 = gold->in(1); |
|
3066 |
Node* gin2 = gold->in(2); |
|
3067 |
Node* fin1 = fix->in(1); |
|
3068 |
Node* fin2 = fix->in(2); |
|
3069 |
bool swapped = false; |
|
3070 |
||
3071 |
if (in_bb(gin1) && in_bb(gin2) && in_bb(fin1) && in_bb(fin1)) { |
|
3072 |
if (_clone_map.idx(gin1->_idx) == _clone_map.idx(fin1->_idx) && |
|
3073 |
_clone_map.idx(gin2->_idx) == _clone_map.idx(fin2->_idx)) { |
|
3074 |
return true; // nothing to fix |
|
3075 |
} |
|
3076 |
if (_clone_map.idx(gin1->_idx) == _clone_map.idx(fin2->_idx) && |
|
3077 |
_clone_map.idx(gin2->_idx) == _clone_map.idx(fin1->_idx)) { |
|
3078 |
fix->swap_edges(1, 2); |
|
3079 |
swapped = true; |
|
3080 |
} |
|
3081 |
} |
|
3082 |
// at least one input comes from outside of bb |
|
3083 |
if (gin1->_idx == fin1->_idx) { |
|
3084 |
return true; // nothing to fix |
|
3085 |
} |
|
3086 |
if (!swapped && (gin1->_idx == fin2->_idx || gin2->_idx == fin1->_idx)) { //swapping is expensive, check condition first |
|
3087 |
fix->swap_edges(1, 2); |
|
3088 |
swapped = true; |
|
3089 |
} |
|
3090 |
||
3091 |
if (swapped) { |
|
3092 |
#ifndef PRODUCT |
|
3093 |
if (_vector_loop_debug) { |
|
3094 |
tty->print_cr("SuperWord::fix_commutative_inputs: fixed node %d", fix->_idx); |
|
3095 |
} |
|
3096 |
#endif |
|
3097 |
return true; |
|
3098 |
} |
|
3099 |
||
3100 |
#ifndef PRODUCT |
|
3101 |
if (TraceSuperWord && Verbose) { |
|
3102 |
tty->print_cr("SuperWord::fix_commutative_inputs: cannot fix node %d", fix->_idx); |
|
3103 |
} |
|
3104 |
#endif |
|
3105 |
return false; |
|
3106 |
} |
|
3107 |
||
3108 |
bool SuperWord::pack_parallel() { |
|
3109 |
#ifndef PRODUCT |
|
3110 |
if (_vector_loop_debug) { |
|
3111 |
tty->print_cr("SuperWord::pack_parallel: START"); |
|
3112 |
} |
|
3113 |
#endif |
|
3114 |
||
3115 |
_packset.clear(); |
|
3116 |
||
3117 |
for (int ii = 0; ii < _iteration_first.length(); ii++) { |
|
3118 |
Node* nd = _iteration_first.at(ii); |
|
3119 |
if (in_bb(nd) && (nd->is_Load() || nd->is_Store() || nd->is_Add() || nd->is_Mul())) { |
|
3120 |
Node_List* pk = new Node_List(); |
|
3121 |
pk->push(nd); |
|
3122 |
for (int gen = 1; gen < _ii_order.length(); ++gen) { |
|
3123 |
for (int kk = 0; kk < _block.length(); kk++) { |
|
3124 |
Node* clone = _block.at(kk); |
|
3125 |
if (_clone_map.idx(clone->_idx) == _clone_map.idx(nd->_idx) && |
|
3126 |
_clone_map.gen(clone->_idx) == _ii_order.at(gen)) { |
|
3127 |
if (nd->is_Add() || nd->is_Mul()) { |
|
3128 |
fix_commutative_inputs(nd, clone); |
|
3129 |
} |
|
3130 |
pk->push(clone); |
|
3131 |
if (pk->size() == 4) { |
|
3132 |
_packset.append(pk); |
|
3133 |
#ifndef PRODUCT |
|
3134 |
if (_vector_loop_debug) { |
|
3135 |
tty->print_cr("SuperWord::pack_parallel: added pack "); |
|
3136 |
pk->dump(); |
|
3137 |
} |
|
3138 |
#endif |
|
3139 |
if (_clone_map.gen(clone->_idx) != _ii_last) { |
|
3140 |
pk = new Node_List(); |
|
3141 |
} |
|
3142 |
} |
|
3143 |
break; |
|
3144 |
} |
|
3145 |
} |
|
3146 |
}//for |
|
3147 |
}//if |
|
3148 |
}//for |
|
3149 |
||
3150 |
#ifndef PRODUCT |
|
3151 |
if (_vector_loop_debug) { |
|
3152 |
tty->print_cr("SuperWord::pack_parallel: END"); |
|
3153 |
} |
|
3154 |
#endif |
|
3155 |
||
3156 |
return true; |
|
3157 |
} |
|
3158 |
||
3159 |
bool SuperWord::hoist_loads_in_graph() { |
|
3160 |
GrowableArray<Node*> loads; |
|
3161 |
||
3162 |
#ifndef PRODUCT |
|
3163 |
if (_vector_loop_debug) { |
|
3164 |
tty->print_cr("SuperWord::hoist_loads_in_graph: total number _mem_slice_head.length() = %d", _mem_slice_head.length()); |
|
3165 |
} |
|
3166 |
#endif |
|
3167 |
||
3168 |
for (int i = 0; i < _mem_slice_head.length(); i++) { |
|
3169 |
Node* n = _mem_slice_head.at(i); |
|
3170 |
if ( !in_bb(n) || !n->is_Phi() || n->bottom_type() != Type::MEMORY) { |
|
3171 |
#ifndef PRODUCT |
|
3172 |
if (TraceSuperWord && Verbose) { |
|
3173 |
tty->print_cr("SuperWord::hoist_loads_in_graph: skipping unexpected node n=%d", n->_idx); |
|
3174 |
} |
|
3175 |
#endif |
|
3176 |
continue; |
|
3177 |
} |
|
3178 |
||
3179 |
#ifndef PRODUCT |
|
3180 |
if (_vector_loop_debug) { |
|
3181 |
tty->print_cr("SuperWord::hoist_loads_in_graph: processing phi %d = _mem_slice_head.at(%d);", n->_idx, i); |
|
3182 |
} |
|
3183 |
#endif |
|
3184 |
||
3185 |
for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { |
|
3186 |
Node* ld = n->fast_out(i); |
|
3187 |
if (ld->is_Load() && ld->as_Load()->in(MemNode::Memory) == n && in_bb(ld)) { |
|
3188 |
for (int i = 0; i < _block.length(); i++) { |
|
3189 |
Node* ld2 = _block.at(i); |
|
3190 |
if (ld2->is_Load() && |
|
3191 |
_clone_map.idx(ld->_idx) == _clone_map.idx(ld2->_idx) && |
|
3192 |
_clone_map.gen(ld->_idx) != _clone_map.gen(ld2->_idx)) { // <= do not collect the first generation ld |
|
3193 |
#ifndef PRODUCT |
|
3194 |
if (_vector_loop_debug) { |
|
3195 |
tty->print_cr("SuperWord::hoist_loads_in_graph: will try to hoist load ld2->_idx=%d, cloned from %d (ld->_idx=%d)", |
|
3196 |
ld2->_idx, _clone_map.idx(ld->_idx), ld->_idx); |
|
3197 |
} |
|
3198 |
#endif |
|
3199 |
// could not do on-the-fly, since iterator is immutable |
|
3200 |
loads.push(ld2); |
|
3201 |
} |
|
3202 |
}// for |
|
3203 |
}//if |
|
3204 |
}//for (DUIterator_Fast imax, |
|
3205 |
}//for (int i = 0; i |
|
3206 |
||
3207 |
for (int i = 0; i < loads.length(); i++) { |
|
3208 |
LoadNode* ld = loads.at(i)->as_Load(); |
|
3209 |
Node* phi = find_phi_for_mem_dep(ld); |
|
3210 |
if (phi != NULL) { |
|
3211 |
#ifndef PRODUCT |
|
3212 |
if (_vector_loop_debug) { |
|
3213 |
tty->print_cr("SuperWord::hoist_loads_in_graph replacing MemNode::Memory(%d) edge in %d with one from %d", |
|
3214 |
MemNode::Memory, ld->_idx, phi->_idx); |
|
3215 |
} |
|
3216 |
#endif |
|
3217 |
_igvn.replace_input_of(ld, MemNode::Memory, phi); |
|
3218 |
} |
|
3219 |
}//for |
|
3220 |
||
3221 |
restart(); // invalidate all basic structures, since we rebuilt the graph |
|
3222 |
||
3223 |
#ifndef PRODUCT |
|
3224 |
if (TraceSuperWord && Verbose) { |
|
3225 |
tty->print_cr("\nSuperWord::hoist_loads_in_graph() the graph was rebuilt, all structures invalidated and need rebuild"); |
|
3226 |
} |
|
3227 |
#endif |
|
3228 |
return true; |
|
3229 |
} |
|
3230 |