author | kvn |
Tue, 26 Jun 2012 09:06:16 -0700 | |
changeset 13111 | e8a578a9b20a |
parent 13108 | 6d27f658925c |
child 13485 | 6c7faa516fc6 |
permissions | -rw-r--r-- |
1 | 1 |
/* |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2 |
* Copyright (c) 2007, 2012, Oracle and/or its affiliates. All rights reserved. |
1 | 3 |
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
4 |
* |
|
5 |
* This code is free software; you can redistribute it and/or modify it |
|
6 |
* under the terms of the GNU General Public License version 2 only, as |
|
7 |
* published by the Free Software Foundation. |
|
8 |
* |
|
9 |
* This code is distributed in the hope that it will be useful, but WITHOUT |
|
10 |
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
|
11 |
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
|
12 |
* version 2 for more details (a copy is included in the LICENSE file that |
|
13 |
* accompanied this code). |
|
14 |
* |
|
15 |
* You should have received a copy of the GNU General Public License version |
|
16 |
* 2 along with this work; if not, write to the Free Software Foundation, |
|
17 |
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
|
18 |
* |
|
5547
f4b087cbb361
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
4428
diff
changeset
|
19 |
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
f4b087cbb361
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
4428
diff
changeset
|
20 |
* or visit www.oracle.com if you need additional information or have any |
f4b087cbb361
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
4428
diff
changeset
|
21 |
* questions. |
1 | 22 |
*/ |
23 |
||
7397 | 24 |
#include "precompiled.hpp" |
25 |
#include "compiler/compileLog.hpp" |
|
26 |
#include "libadt/vectset.hpp" |
|
27 |
#include "memory/allocation.inline.hpp" |
|
28 |
#include "opto/addnode.hpp" |
|
29 |
#include "opto/callnode.hpp" |
|
30 |
#include "opto/divnode.hpp" |
|
31 |
#include "opto/matcher.hpp" |
|
32 |
#include "opto/memnode.hpp" |
|
33 |
#include "opto/mulnode.hpp" |
|
34 |
#include "opto/opcodes.hpp" |
|
35 |
#include "opto/superword.hpp" |
|
36 |
#include "opto/vectornode.hpp" |
|
1 | 37 |
|
38 |
// |
|
39 |
// S U P E R W O R D T R A N S F O R M |
|
40 |
//============================================================================= |
|
41 |
||
42 |
//------------------------------SuperWord--------------------------- |
|
43 |
SuperWord::SuperWord(PhaseIdealLoop* phase) : |
|
44 |
_phase(phase), |
|
45 |
_igvn(phase->_igvn), |
|
46 |
_arena(phase->C->comp_arena()), |
|
47 |
_packset(arena(), 8, 0, NULL), // packs for the current block |
|
48 |
_bb_idx(arena(), (int)(1.10 * phase->C->unique()), 0, 0), // node idx to index in bb |
|
49 |
_block(arena(), 8, 0, NULL), // nodes in current block |
|
50 |
_data_entry(arena(), 8, 0, NULL), // nodes with all inputs from outside |
|
51 |
_mem_slice_head(arena(), 8, 0, NULL), // memory slice heads |
|
52 |
_mem_slice_tail(arena(), 8, 0, NULL), // memory slice tails |
|
53 |
_node_info(arena(), 8, 0, SWNodeInfo::initial), // info needed per node |
|
54 |
_align_to_ref(NULL), // memory reference to align vectors to |
|
55 |
_disjoint_ptrs(arena(), 8, 0, OrderedPair::initial), // runtime disambiguated pointer pairs |
|
56 |
_dg(_arena), // dependence graph |
|
57 |
_visited(arena()), // visited node set |
|
58 |
_post_visited(arena()), // post visited node set |
|
59 |
_n_idx_list(arena(), 8), // scratch list of (node,index) pairs |
|
60 |
_stk(arena(), 8, 0, NULL), // scratch stack of nodes |
|
61 |
_nlist(arena(), 8, 0, NULL), // scratch list of nodes |
|
62 |
_lpt(NULL), // loop tree node |
|
63 |
_lp(NULL), // LoopNode |
|
64 |
_bb(NULL), // basic block |
|
65 |
_iv(NULL) // induction var |
|
66 |
{} |
|
67 |
||
68 |
//------------------------------transform_loop--------------------------- |
|
69 |
void SuperWord::transform_loop(IdealLoopTree* lpt) { |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
70 |
assert(UseSuperWord, "should be"); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
71 |
// Do vectors exist on this architecture? |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
72 |
if (Matcher::vector_width_in_bytes(T_BYTE) < 2) return; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
73 |
|
1 | 74 |
assert(lpt->_head->is_CountedLoop(), "must be"); |
75 |
CountedLoopNode *cl = lpt->_head->as_CountedLoop(); |
|
76 |
||
10263
fa58671dde31
7077439: Possible reference through NULL in loopPredicate.cpp:726
kvn
parents:
10255
diff
changeset
|
77 |
if (!cl->is_valid_counted_loop()) return; // skip malformed counted loop |
fa58671dde31
7077439: Possible reference through NULL in loopPredicate.cpp:726
kvn
parents:
10255
diff
changeset
|
78 |
|
1 | 79 |
if (!cl->is_main_loop() ) return; // skip normal, pre, and post loops |
80 |
||
81 |
// Check for no control flow in body (other than exit) |
|
82 |
Node *cl_exit = cl->loopexit(); |
|
83 |
if (cl_exit->in(0) != lpt->_head) return; |
|
84 |
||
352
6d8e1aa5834e
6646020: assert(in_bb(n),"must be in block") in -Xcomp mode
never
parents:
245
diff
changeset
|
85 |
// Make sure the are no extra control users of the loop backedge |
6d8e1aa5834e
6646020: assert(in_bb(n),"must be in block") in -Xcomp mode
never
parents:
245
diff
changeset
|
86 |
if (cl->back_control()->outcnt() != 1) { |
6d8e1aa5834e
6646020: assert(in_bb(n),"must be in block") in -Xcomp mode
never
parents:
245
diff
changeset
|
87 |
return; |
6d8e1aa5834e
6646020: assert(in_bb(n),"must be in block") in -Xcomp mode
never
parents:
245
diff
changeset
|
88 |
} |
6d8e1aa5834e
6646020: assert(in_bb(n),"must be in block") in -Xcomp mode
never
parents:
245
diff
changeset
|
89 |
|
1 | 90 |
// Check for pre-loop ending with CountedLoopEnd(Bool(Cmp(x,Opaque1(limit)))) |
91 |
CountedLoopEndNode* pre_end = get_pre_loop_end(cl); |
|
92 |
if (pre_end == NULL) return; |
|
93 |
Node *pre_opaq1 = pre_end->limit(); |
|
94 |
if (pre_opaq1->Opcode() != Op_Opaque1) return; |
|
95 |
||
96 |
init(); // initialize data structures |
|
97 |
||
98 |
set_lpt(lpt); |
|
99 |
set_lp(cl); |
|
100 |
||
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
101 |
// For now, define one block which is the entire loop body |
1 | 102 |
set_bb(cl); |
103 |
||
104 |
assert(_packset.length() == 0, "packset must be empty"); |
|
105 |
SLP_extract(); |
|
106 |
} |
|
107 |
||
108 |
//------------------------------SLP_extract--------------------------- |
|
109 |
// Extract the superword level parallelism |
|
110 |
// |
|
111 |
// 1) A reverse post-order of nodes in the block is constructed. By scanning |
|
112 |
// this list from first to last, all definitions are visited before their uses. |
|
113 |
// |
|
114 |
// 2) A point-to-point dependence graph is constructed between memory references. |
|
115 |
// This simplies the upcoming "independence" checker. |
|
116 |
// |
|
117 |
// 3) The maximum depth in the node graph from the beginning of the block |
|
118 |
// to each node is computed. This is used to prune the graph search |
|
119 |
// in the independence checker. |
|
120 |
// |
|
121 |
// 4) For integer types, the necessary bit width is propagated backwards |
|
122 |
// from stores to allow packed operations on byte, char, and short |
|
123 |
// integers. This reverses the promotion to type "int" that javac |
|
124 |
// did for operations like: char c1,c2,c3; c1 = c2 + c3. |
|
125 |
// |
|
126 |
// 5) One of the memory references is picked to be an aligned vector reference. |
|
127 |
// The pre-loop trip count is adjusted to align this reference in the |
|
128 |
// unrolled body. |
|
129 |
// |
|
130 |
// 6) The initial set of pack pairs is seeded with memory references. |
|
131 |
// |
|
132 |
// 7) The set of pack pairs is extended by following use->def and def->use links. |
|
133 |
// |
|
134 |
// 8) The pairs are combined into vector sized packs. |
|
135 |
// |
|
136 |
// 9) Reorder the memory slices to co-locate members of the memory packs. |
|
137 |
// |
|
138 |
// 10) Generate ideal vector nodes for the final set of packs and where necessary, |
|
139 |
// inserting scalar promotion, vector creation from multiple scalars, and |
|
140 |
// extraction of scalar values from vectors. |
|
141 |
// |
|
142 |
void SuperWord::SLP_extract() { |
|
143 |
||
144 |
// Ready the block |
|
145 |
||
146 |
construct_bb(); |
|
147 |
||
148 |
dependence_graph(); |
|
149 |
||
150 |
compute_max_depth(); |
|
151 |
||
152 |
compute_vector_element_type(); |
|
153 |
||
154 |
// Attempt vectorization |
|
155 |
||
156 |
find_adjacent_refs(); |
|
157 |
||
158 |
extend_packlist(); |
|
159 |
||
160 |
combine_packs(); |
|
161 |
||
162 |
construct_my_pack_map(); |
|
163 |
||
164 |
filter_packs(); |
|
165 |
||
166 |
schedule(); |
|
167 |
||
168 |
output(); |
|
169 |
} |
|
170 |
||
171 |
//------------------------------find_adjacent_refs--------------------------- |
|
172 |
// Find the adjacent memory references and create pack pairs for them. |
|
173 |
// This is the initial set of packs that will then be extended by |
|
174 |
// following use->def and def->use links. The align positions are |
|
175 |
// assigned relative to the reference "align_to_ref" |
|
176 |
void SuperWord::find_adjacent_refs() { |
|
177 |
// Get list of memory operations |
|
178 |
Node_List memops; |
|
179 |
for (int i = 0; i < _block.length(); i++) { |
|
180 |
Node* n = _block.at(i); |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
181 |
if (n->is_Mem() && !n->is_LoadStore() && in_bb(n) && |
202
dc13bf0e5d5d
6633953: type2aelembytes{T_ADDRESS} should be 8 bytes in 64 bit VM
kvn
parents:
1
diff
changeset
|
182 |
is_java_primitive(n->as_Mem()->memory_type())) { |
1 | 183 |
int align = memory_alignment(n->as_Mem(), 0); |
184 |
if (align != bottom_align) { |
|
185 |
memops.push(n); |
|
186 |
} |
|
187 |
} |
|
188 |
} |
|
189 |
||
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
190 |
Node_List align_to_refs; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
191 |
int best_iv_adjustment = 0; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
192 |
MemNode* best_align_to_mem_ref = NULL; |
1 | 193 |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
194 |
while (memops.size() != 0) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
195 |
// Find a memory reference to align to. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
196 |
MemNode* mem_ref = find_align_to_ref(memops); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
197 |
if (mem_ref == NULL) break; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
198 |
align_to_refs.push(mem_ref); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
199 |
int iv_adjustment = get_iv_adjustment(mem_ref); |
1 | 200 |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
201 |
if (best_align_to_mem_ref == NULL) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
202 |
// Set memory reference which is the best from all memory operations |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
203 |
// to be used for alignment. The pre-loop trip count is modified to align |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
204 |
// this reference to a vector-aligned address. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
205 |
best_align_to_mem_ref = mem_ref; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
206 |
best_iv_adjustment = iv_adjustment; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
207 |
} |
1 | 208 |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
209 |
SWPointer align_to_ref_p(mem_ref, this); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
210 |
// Set alignment relative to "align_to_ref" for all related memory operations. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
211 |
for (int i = memops.size() - 1; i >= 0; i--) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
212 |
MemNode* s = memops.at(i)->as_Mem(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
213 |
if (isomorphic(s, mem_ref)) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
214 |
SWPointer p2(s, this); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
215 |
if (p2.comparable(align_to_ref_p)) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
216 |
int align = memory_alignment(s, iv_adjustment); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
217 |
set_alignment(s, align); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
218 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
219 |
} |
1 | 220 |
} |
221 |
||
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
222 |
// Create initial pack pairs of memory operations for which |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
223 |
// alignment is set and vectors will be aligned. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
224 |
bool create_pack = true; |
13108
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
225 |
if (memory_alignment(mem_ref, best_iv_adjustment) == 0) { |
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
226 |
if (!Matcher::misaligned_vectors_ok()) { |
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
227 |
int vw = vector_width(mem_ref); |
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
228 |
int vw_best = vector_width(best_align_to_mem_ref); |
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
229 |
if (vw > vw_best) { |
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
230 |
// Do not vectorize a memory access with more elements per vector |
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
231 |
// if unaligned memory access is not allowed because number of |
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
232 |
// iterations in pre-loop will be not enough to align it. |
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
233 |
create_pack = false; |
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
234 |
} |
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
235 |
} |
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
236 |
} else { |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
237 |
if (same_velt_type(mem_ref, best_align_to_mem_ref)) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
238 |
// Can't allow vectorization of unaligned memory accesses with the |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
239 |
// same type since it could be overlapped accesses to the same array. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
240 |
create_pack = false; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
241 |
} else { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
242 |
// Allow independent (different type) unaligned memory operations |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
243 |
// if HW supports them. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
244 |
if (!Matcher::misaligned_vectors_ok()) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
245 |
create_pack = false; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
246 |
} else { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
247 |
// Check if packs of the same memory type but |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
248 |
// with a different alignment were created before. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
249 |
for (uint i = 0; i < align_to_refs.size(); i++) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
250 |
MemNode* mr = align_to_refs.at(i)->as_Mem(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
251 |
if (same_velt_type(mr, mem_ref) && |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
252 |
memory_alignment(mr, iv_adjustment) != 0) |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
253 |
create_pack = false; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
254 |
} |
1 | 255 |
} |
256 |
} |
|
257 |
} |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
258 |
if (create_pack) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
259 |
for (uint i = 0; i < memops.size(); i++) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
260 |
Node* s1 = memops.at(i); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
261 |
int align = alignment(s1); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
262 |
if (align == top_align) continue; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
263 |
for (uint j = 0; j < memops.size(); j++) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
264 |
Node* s2 = memops.at(j); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
265 |
if (alignment(s2) == top_align) continue; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
266 |
if (s1 != s2 && are_adjacent_refs(s1, s2)) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
267 |
if (stmts_can_pack(s1, s2, align)) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
268 |
Node_List* pair = new Node_List(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
269 |
pair->push(s1); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
270 |
pair->push(s2); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
271 |
_packset.append(pair); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
272 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
273 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
274 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
275 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
276 |
} else { // Don't create unaligned pack |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
277 |
// First, remove remaining memory ops of the same type from the list. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
278 |
for (int i = memops.size() - 1; i >= 0; i--) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
279 |
MemNode* s = memops.at(i)->as_Mem(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
280 |
if (same_velt_type(s, mem_ref)) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
281 |
memops.remove(i); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
282 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
283 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
284 |
|
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
285 |
// Second, remove already constructed packs of the same type. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
286 |
for (int i = _packset.length() - 1; i >= 0; i--) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
287 |
Node_List* p = _packset.at(i); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
288 |
MemNode* s = p->at(0)->as_Mem(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
289 |
if (same_velt_type(s, mem_ref)) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
290 |
remove_pack_at(i); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
291 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
292 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
293 |
|
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
294 |
// If needed find the best memory reference for loop alignment again. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
295 |
if (same_velt_type(mem_ref, best_align_to_mem_ref)) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
296 |
// Put memory ops from remaining packs back on memops list for |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
297 |
// the best alignment search. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
298 |
uint orig_msize = memops.size(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
299 |
for (int i = 0; i < _packset.length(); i++) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
300 |
Node_List* p = _packset.at(i); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
301 |
MemNode* s = p->at(0)->as_Mem(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
302 |
assert(!same_velt_type(s, mem_ref), "sanity"); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
303 |
memops.push(s); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
304 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
305 |
MemNode* best_align_to_mem_ref = find_align_to_ref(memops); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
306 |
if (best_align_to_mem_ref == NULL) break; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
307 |
best_iv_adjustment = get_iv_adjustment(best_align_to_mem_ref); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
308 |
// Restore list. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
309 |
while (memops.size() > orig_msize) |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
310 |
(void)memops.pop(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
311 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
312 |
} // unaligned memory accesses |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
313 |
|
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
314 |
// Remove used mem nodes. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
315 |
for (int i = memops.size() - 1; i >= 0; i--) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
316 |
MemNode* m = memops.at(i)->as_Mem(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
317 |
if (alignment(m) != top_align) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
318 |
memops.remove(i); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
319 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
320 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
321 |
|
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
322 |
} // while (memops.size() != 0 |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
323 |
set_align_to_ref(best_align_to_mem_ref); |
1 | 324 |
|
325 |
#ifndef PRODUCT |
|
326 |
if (TraceSuperWord) { |
|
327 |
tty->print_cr("\nAfter find_adjacent_refs"); |
|
328 |
print_packset(); |
|
329 |
} |
|
330 |
#endif |
|
331 |
} |
|
332 |
||
333 |
//------------------------------find_align_to_ref--------------------------- |
|
334 |
// Find a memory reference to align the loop induction variable to. |
|
335 |
// Looks first at stores then at loads, looking for a memory reference |
|
336 |
// with the largest number of references similar to it. |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
337 |
MemNode* SuperWord::find_align_to_ref(Node_List &memops) { |
1 | 338 |
GrowableArray<int> cmp_ct(arena(), memops.size(), memops.size(), 0); |
339 |
||
340 |
// Count number of comparable memory ops |
|
341 |
for (uint i = 0; i < memops.size(); i++) { |
|
342 |
MemNode* s1 = memops.at(i)->as_Mem(); |
|
343 |
SWPointer p1(s1, this); |
|
344 |
// Discard if pre loop can't align this reference |
|
345 |
if (!ref_is_alignable(p1)) { |
|
346 |
*cmp_ct.adr_at(i) = 0; |
|
347 |
continue; |
|
348 |
} |
|
349 |
for (uint j = i+1; j < memops.size(); j++) { |
|
350 |
MemNode* s2 = memops.at(j)->as_Mem(); |
|
351 |
if (isomorphic(s1, s2)) { |
|
352 |
SWPointer p2(s2, this); |
|
353 |
if (p1.comparable(p2)) { |
|
354 |
(*cmp_ct.adr_at(i))++; |
|
355 |
(*cmp_ct.adr_at(j))++; |
|
356 |
} |
|
357 |
} |
|
358 |
} |
|
359 |
} |
|
360 |
||
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
361 |
// Find Store (or Load) with the greatest number of "comparable" references, |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
362 |
// biggest vector size, smallest data size and smallest iv offset. |
1 | 363 |
int max_ct = 0; |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
364 |
int max_vw = 0; |
1 | 365 |
int max_idx = -1; |
366 |
int min_size = max_jint; |
|
367 |
int min_iv_offset = max_jint; |
|
368 |
for (uint j = 0; j < memops.size(); j++) { |
|
369 |
MemNode* s = memops.at(j)->as_Mem(); |
|
370 |
if (s->is_Store()) { |
|
13108
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
371 |
int vw = vector_width_in_bytes(s); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
372 |
assert(vw > 1, "sanity"); |
1 | 373 |
SWPointer p(s, this); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
374 |
if (cmp_ct.at(j) > max_ct || |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
375 |
cmp_ct.at(j) == max_ct && |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
376 |
(vw > max_vw || |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
377 |
vw == max_vw && |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
378 |
(data_size(s) < min_size || |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
379 |
data_size(s) == min_size && |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
380 |
(p.offset_in_bytes() < min_iv_offset)))) { |
1 | 381 |
max_ct = cmp_ct.at(j); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
382 |
max_vw = vw; |
1 | 383 |
max_idx = j; |
384 |
min_size = data_size(s); |
|
385 |
min_iv_offset = p.offset_in_bytes(); |
|
386 |
} |
|
387 |
} |
|
388 |
} |
|
389 |
// If no stores, look at loads |
|
390 |
if (max_ct == 0) { |
|
391 |
for (uint j = 0; j < memops.size(); j++) { |
|
392 |
MemNode* s = memops.at(j)->as_Mem(); |
|
393 |
if (s->is_Load()) { |
|
13108
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
394 |
int vw = vector_width_in_bytes(s); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
395 |
assert(vw > 1, "sanity"); |
1 | 396 |
SWPointer p(s, this); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
397 |
if (cmp_ct.at(j) > max_ct || |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
398 |
cmp_ct.at(j) == max_ct && |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
399 |
(vw > max_vw || |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
400 |
vw == max_vw && |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
401 |
(data_size(s) < min_size || |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
402 |
data_size(s) == min_size && |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
403 |
(p.offset_in_bytes() < min_iv_offset)))) { |
1 | 404 |
max_ct = cmp_ct.at(j); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
405 |
max_vw = vw; |
1 | 406 |
max_idx = j; |
407 |
min_size = data_size(s); |
|
408 |
min_iv_offset = p.offset_in_bytes(); |
|
409 |
} |
|
410 |
} |
|
411 |
} |
|
412 |
} |
|
413 |
||
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
414 |
#ifdef ASSERT |
1 | 415 |
if (TraceSuperWord && Verbose) { |
416 |
tty->print_cr("\nVector memops after find_align_to_refs"); |
|
417 |
for (uint i = 0; i < memops.size(); i++) { |
|
418 |
MemNode* s = memops.at(i)->as_Mem(); |
|
419 |
s->dump(); |
|
420 |
} |
|
421 |
} |
|
422 |
#endif |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
423 |
|
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
424 |
if (max_ct > 0) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
425 |
#ifdef ASSERT |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
426 |
if (TraceSuperWord) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
427 |
tty->print("\nVector align to node: "); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
428 |
memops.at(max_idx)->as_Mem()->dump(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
429 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
430 |
#endif |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
431 |
return memops.at(max_idx)->as_Mem(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
432 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
433 |
return NULL; |
1 | 434 |
} |
435 |
||
436 |
//------------------------------ref_is_alignable--------------------------- |
|
437 |
// Can the preloop align the reference to position zero in the vector? |
|
438 |
bool SuperWord::ref_is_alignable(SWPointer& p) { |
|
439 |
if (!p.has_iv()) { |
|
440 |
return true; // no induction variable |
|
441 |
} |
|
442 |
CountedLoopEndNode* pre_end = get_pre_loop_end(lp()->as_CountedLoop()); |
|
443 |
assert(pre_end->stride_is_con(), "pre loop stride is constant"); |
|
444 |
int preloop_stride = pre_end->stride_con(); |
|
445 |
||
446 |
int span = preloop_stride * p.scale_in_bytes(); |
|
447 |
||
448 |
// Stride one accesses are alignable. |
|
449 |
if (ABS(span) == p.memory_size()) |
|
450 |
return true; |
|
451 |
||
452 |
// If initial offset from start of object is computable, |
|
453 |
// compute alignment within the vector. |
|
13108
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
454 |
int vw = vector_width_in_bytes(p.mem()); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
455 |
assert(vw > 1, "sanity"); |
1 | 456 |
if (vw % span == 0) { |
457 |
Node* init_nd = pre_end->init_trip(); |
|
458 |
if (init_nd->is_Con() && p.invar() == NULL) { |
|
459 |
int init = init_nd->bottom_type()->is_int()->get_con(); |
|
460 |
||
461 |
int init_offset = init * p.scale_in_bytes() + p.offset_in_bytes(); |
|
462 |
assert(init_offset >= 0, "positive offset from object start"); |
|
463 |
||
464 |
if (span > 0) { |
|
465 |
return (vw - (init_offset % vw)) % span == 0; |
|
466 |
} else { |
|
467 |
assert(span < 0, "nonzero stride * scale"); |
|
468 |
return (init_offset % vw) % -span == 0; |
|
469 |
} |
|
470 |
} |
|
471 |
} |
|
472 |
return false; |
|
473 |
} |
|
474 |
||
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
475 |
//---------------------------get_iv_adjustment--------------------------- |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
476 |
// Calculate loop's iv adjustment for this memory ops. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
477 |
int SuperWord::get_iv_adjustment(MemNode* mem_ref) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
478 |
SWPointer align_to_ref_p(mem_ref, this); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
479 |
int offset = align_to_ref_p.offset_in_bytes(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
480 |
int scale = align_to_ref_p.scale_in_bytes(); |
13108
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
481 |
int vw = vector_width_in_bytes(mem_ref); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
482 |
assert(vw > 1, "sanity"); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
483 |
int stride_sign = (scale * iv_stride()) > 0 ? 1 : -1; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
484 |
int iv_adjustment = (stride_sign * vw - (offset % vw)) % vw; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
485 |
|
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
486 |
#ifndef PRODUCT |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
487 |
if (TraceSuperWord) |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
488 |
tty->print_cr("\noffset = %d iv_adjust = %d elt_size = %d scale = %d iv_stride = %d vect_size %d", |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
489 |
offset, iv_adjustment, align_to_ref_p.memory_size(), scale, iv_stride(), vw); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
490 |
#endif |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
491 |
return iv_adjustment; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
492 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
493 |
|
1 | 494 |
//---------------------------dependence_graph--------------------------- |
495 |
// Construct dependency graph. |
|
496 |
// Add dependence edges to load/store nodes for memory dependence |
|
497 |
// A.out()->DependNode.in(1) and DependNode.out()->B.prec(x) |
|
498 |
void SuperWord::dependence_graph() { |
|
499 |
// First, assign a dependence node to each memory node |
|
500 |
for (int i = 0; i < _block.length(); i++ ) { |
|
501 |
Node *n = _block.at(i); |
|
502 |
if (n->is_Mem() || n->is_Phi() && n->bottom_type() == Type::MEMORY) { |
|
503 |
_dg.make_node(n); |
|
504 |
} |
|
505 |
} |
|
506 |
||
507 |
// For each memory slice, create the dependences |
|
508 |
for (int i = 0; i < _mem_slice_head.length(); i++) { |
|
509 |
Node* n = _mem_slice_head.at(i); |
|
510 |
Node* n_tail = _mem_slice_tail.at(i); |
|
511 |
||
512 |
// Get slice in predecessor order (last is first) |
|
513 |
mem_slice_preds(n_tail, n, _nlist); |
|
514 |
||
515 |
// Make the slice dependent on the root |
|
516 |
DepMem* slice = _dg.dep(n); |
|
517 |
_dg.make_edge(_dg.root(), slice); |
|
518 |
||
519 |
// Create a sink for the slice |
|
520 |
DepMem* slice_sink = _dg.make_node(NULL); |
|
521 |
_dg.make_edge(slice_sink, _dg.tail()); |
|
522 |
||
523 |
// Now visit each pair of memory ops, creating the edges |
|
524 |
for (int j = _nlist.length() - 1; j >= 0 ; j--) { |
|
525 |
Node* s1 = _nlist.at(j); |
|
526 |
||
527 |
// If no dependency yet, use slice |
|
528 |
if (_dg.dep(s1)->in_cnt() == 0) { |
|
529 |
_dg.make_edge(slice, s1); |
|
530 |
} |
|
531 |
SWPointer p1(s1->as_Mem(), this); |
|
532 |
bool sink_dependent = true; |
|
533 |
for (int k = j - 1; k >= 0; k--) { |
|
534 |
Node* s2 = _nlist.at(k); |
|
535 |
if (s1->is_Load() && s2->is_Load()) |
|
536 |
continue; |
|
537 |
SWPointer p2(s2->as_Mem(), this); |
|
538 |
||
539 |
int cmp = p1.cmp(p2); |
|
540 |
if (SuperWordRTDepCheck && |
|
541 |
p1.base() != p2.base() && p1.valid() && p2.valid()) { |
|
542 |
// Create a runtime check to disambiguate |
|
543 |
OrderedPair pp(p1.base(), p2.base()); |
|
544 |
_disjoint_ptrs.append_if_missing(pp); |
|
545 |
} else if (!SWPointer::not_equal(cmp)) { |
|
546 |
// Possibly same address |
|
547 |
_dg.make_edge(s1, s2); |
|
548 |
sink_dependent = false; |
|
549 |
} |
|
550 |
} |
|
551 |
if (sink_dependent) { |
|
552 |
_dg.make_edge(s1, slice_sink); |
|
553 |
} |
|
554 |
} |
|
555 |
#ifndef PRODUCT |
|
556 |
if (TraceSuperWord) { |
|
557 |
tty->print_cr("\nDependence graph for slice: %d", n->_idx); |
|
558 |
for (int q = 0; q < _nlist.length(); q++) { |
|
559 |
_dg.print(_nlist.at(q)); |
|
560 |
} |
|
561 |
tty->cr(); |
|
562 |
} |
|
563 |
#endif |
|
564 |
_nlist.clear(); |
|
565 |
} |
|
566 |
||
567 |
#ifndef PRODUCT |
|
568 |
if (TraceSuperWord) { |
|
569 |
tty->print_cr("\ndisjoint_ptrs: %s", _disjoint_ptrs.length() > 0 ? "" : "NONE"); |
|
570 |
for (int r = 0; r < _disjoint_ptrs.length(); r++) { |
|
571 |
_disjoint_ptrs.at(r).print(); |
|
572 |
tty->cr(); |
|
573 |
} |
|
574 |
tty->cr(); |
|
575 |
} |
|
576 |
#endif |
|
577 |
} |
|
578 |
||
579 |
//---------------------------mem_slice_preds--------------------------- |
|
580 |
// Return a memory slice (node list) in predecessor order starting at "start" |
|
581 |
void SuperWord::mem_slice_preds(Node* start, Node* stop, GrowableArray<Node*> &preds) { |
|
582 |
assert(preds.length() == 0, "start empty"); |
|
583 |
Node* n = start; |
|
584 |
Node* prev = NULL; |
|
585 |
while (true) { |
|
586 |
assert(in_bb(n), "must be in block"); |
|
587 |
for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { |
|
588 |
Node* out = n->fast_out(i); |
|
589 |
if (out->is_Load()) { |
|
590 |
if (in_bb(out)) { |
|
591 |
preds.push(out); |
|
592 |
} |
|
593 |
} else { |
|
594 |
// FIXME |
|
595 |
if (out->is_MergeMem() && !in_bb(out)) { |
|
596 |
// Either unrolling is causing a memory edge not to disappear, |
|
597 |
// or need to run igvn.optimize() again before SLP |
|
598 |
} else if (out->is_Phi() && out->bottom_type() == Type::MEMORY && !in_bb(out)) { |
|
599 |
// Ditto. Not sure what else to check further. |
|
2334 | 600 |
} else if (out->Opcode() == Op_StoreCM && out->in(MemNode::OopStore) == n) { |
1 | 601 |
// StoreCM has an input edge used as a precedence edge. |
602 |
// Maybe an issue when oop stores are vectorized. |
|
603 |
} else { |
|
604 |
assert(out == prev || prev == NULL, "no branches off of store slice"); |
|
605 |
} |
|
606 |
} |
|
607 |
} |
|
608 |
if (n == stop) break; |
|
609 |
preds.push(n); |
|
610 |
prev = n; |
|
611 |
n = n->in(MemNode::Memory); |
|
612 |
} |
|
613 |
} |
|
614 |
||
615 |
//------------------------------stmts_can_pack--------------------------- |
|
2131 | 616 |
// Can s1 and s2 be in a pack with s1 immediately preceding s2 and |
1 | 617 |
// s1 aligned at "align" |
618 |
bool SuperWord::stmts_can_pack(Node* s1, Node* s2, int align) { |
|
3906
6767b0c66883
6879921: CTW failure jdk6_18/hotspot/src/share/vm/utilities/globalDefinitions.cpp:268
cfang
parents:
3904
diff
changeset
|
619 |
|
6767b0c66883
6879921: CTW failure jdk6_18/hotspot/src/share/vm/utilities/globalDefinitions.cpp:268
cfang
parents:
3904
diff
changeset
|
620 |
// Do not use superword for non-primitives |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
621 |
BasicType bt1 = velt_basic_type(s1); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
622 |
BasicType bt2 = velt_basic_type(s2); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
623 |
if(!is_java_primitive(bt1) || !is_java_primitive(bt2)) |
3906
6767b0c66883
6879921: CTW failure jdk6_18/hotspot/src/share/vm/utilities/globalDefinitions.cpp:268
cfang
parents:
3904
diff
changeset
|
624 |
return false; |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
625 |
if (Matcher::max_vector_size(bt1) < 2) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
626 |
return false; // No vectors for this type |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
627 |
} |
3906
6767b0c66883
6879921: CTW failure jdk6_18/hotspot/src/share/vm/utilities/globalDefinitions.cpp:268
cfang
parents:
3904
diff
changeset
|
628 |
|
1 | 629 |
if (isomorphic(s1, s2)) { |
630 |
if (independent(s1, s2)) { |
|
631 |
if (!exists_at(s1, 0) && !exists_at(s2, 1)) { |
|
632 |
if (!s1->is_Mem() || are_adjacent_refs(s1, s2)) { |
|
633 |
int s1_align = alignment(s1); |
|
634 |
int s2_align = alignment(s2); |
|
635 |
if (s1_align == top_align || s1_align == align) { |
|
636 |
if (s2_align == top_align || s2_align == align + data_size(s1)) { |
|
637 |
return true; |
|
638 |
} |
|
639 |
} |
|
640 |
} |
|
641 |
} |
|
642 |
} |
|
643 |
} |
|
644 |
return false; |
|
645 |
} |
|
646 |
||
647 |
//------------------------------exists_at--------------------------- |
|
648 |
// Does s exist in a pack at position pos? |
|
649 |
bool SuperWord::exists_at(Node* s, uint pos) { |
|
650 |
for (int i = 0; i < _packset.length(); i++) { |
|
651 |
Node_List* p = _packset.at(i); |
|
652 |
if (p->at(pos) == s) { |
|
653 |
return true; |
|
654 |
} |
|
655 |
} |
|
656 |
return false; |
|
657 |
} |
|
658 |
||
659 |
//------------------------------are_adjacent_refs--------------------------- |
|
660 |
// Is s1 immediately before s2 in memory? |
|
661 |
bool SuperWord::are_adjacent_refs(Node* s1, Node* s2) { |
|
662 |
if (!s1->is_Mem() || !s2->is_Mem()) return false; |
|
663 |
if (!in_bb(s1) || !in_bb(s2)) return false; |
|
5708 | 664 |
|
665 |
// Do not use superword for non-primitives |
|
666 |
if (!is_java_primitive(s1->as_Mem()->memory_type()) || |
|
667 |
!is_java_primitive(s2->as_Mem()->memory_type())) { |
|
668 |
return false; |
|
669 |
} |
|
670 |
||
1 | 671 |
// FIXME - co_locate_pack fails on Stores in different mem-slices, so |
672 |
// only pack memops that are in the same alias set until that's fixed. |
|
673 |
if (_phase->C->get_alias_index(s1->as_Mem()->adr_type()) != |
|
674 |
_phase->C->get_alias_index(s2->as_Mem()->adr_type())) |
|
675 |
return false; |
|
676 |
SWPointer p1(s1->as_Mem(), this); |
|
677 |
SWPointer p2(s2->as_Mem(), this); |
|
678 |
if (p1.base() != p2.base() || !p1.comparable(p2)) return false; |
|
679 |
int diff = p2.offset_in_bytes() - p1.offset_in_bytes(); |
|
680 |
return diff == data_size(s1); |
|
681 |
} |
|
682 |
||
683 |
//------------------------------isomorphic--------------------------- |
|
684 |
// Are s1 and s2 similar? |
|
685 |
bool SuperWord::isomorphic(Node* s1, Node* s2) { |
|
686 |
if (s1->Opcode() != s2->Opcode()) return false; |
|
687 |
if (s1->req() != s2->req()) return false; |
|
688 |
if (s1->in(0) != s2->in(0)) return false; |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
689 |
if (!same_velt_type(s1, s2)) return false; |
1 | 690 |
return true; |
691 |
} |
|
692 |
||
693 |
//------------------------------independent--------------------------- |
|
694 |
// Is there no data path from s1 to s2 or s2 to s1? |
|
695 |
bool SuperWord::independent(Node* s1, Node* s2) { |
|
696 |
// assert(s1->Opcode() == s2->Opcode(), "check isomorphic first"); |
|
697 |
int d1 = depth(s1); |
|
698 |
int d2 = depth(s2); |
|
699 |
if (d1 == d2) return s1 != s2; |
|
700 |
Node* deep = d1 > d2 ? s1 : s2; |
|
701 |
Node* shallow = d1 > d2 ? s2 : s1; |
|
702 |
||
703 |
visited_clear(); |
|
704 |
||
705 |
return independent_path(shallow, deep); |
|
706 |
} |
|
707 |
||
708 |
//------------------------------independent_path------------------------------ |
|
709 |
// Helper for independent |
|
710 |
bool SuperWord::independent_path(Node* shallow, Node* deep, uint dp) { |
|
711 |
if (dp >= 1000) return false; // stop deep recursion |
|
712 |
visited_set(deep); |
|
713 |
int shal_depth = depth(shallow); |
|
714 |
assert(shal_depth <= depth(deep), "must be"); |
|
715 |
for (DepPreds preds(deep, _dg); !preds.done(); preds.next()) { |
|
716 |
Node* pred = preds.current(); |
|
717 |
if (in_bb(pred) && !visited_test(pred)) { |
|
718 |
if (shallow == pred) { |
|
719 |
return false; |
|
720 |
} |
|
721 |
if (shal_depth < depth(pred) && !independent_path(shallow, pred, dp+1)) { |
|
722 |
return false; |
|
723 |
} |
|
724 |
} |
|
725 |
} |
|
726 |
return true; |
|
727 |
} |
|
728 |
||
729 |
//------------------------------set_alignment--------------------------- |
|
730 |
void SuperWord::set_alignment(Node* s1, Node* s2, int align) { |
|
731 |
set_alignment(s1, align); |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
732 |
if (align == top_align || align == bottom_align) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
733 |
set_alignment(s2, align); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
734 |
} else { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
735 |
set_alignment(s2, align + data_size(s1)); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
736 |
} |
1 | 737 |
} |
738 |
||
739 |
//------------------------------data_size--------------------------- |
|
740 |
int SuperWord::data_size(Node* s) { |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
741 |
int bsize = type2aelembytes(velt_basic_type(s)); |
1 | 742 |
assert(bsize != 0, "valid size"); |
743 |
return bsize; |
|
744 |
} |
|
745 |
||
746 |
//------------------------------extend_packlist--------------------------- |
|
747 |
// Extend packset by following use->def and def->use links from pack members. |
|
748 |
void SuperWord::extend_packlist() { |
|
749 |
bool changed; |
|
750 |
do { |
|
751 |
changed = false; |
|
752 |
for (int i = 0; i < _packset.length(); i++) { |
|
753 |
Node_List* p = _packset.at(i); |
|
754 |
changed |= follow_use_defs(p); |
|
755 |
changed |= follow_def_uses(p); |
|
756 |
} |
|
757 |
} while (changed); |
|
758 |
||
759 |
#ifndef PRODUCT |
|
760 |
if (TraceSuperWord) { |
|
761 |
tty->print_cr("\nAfter extend_packlist"); |
|
762 |
print_packset(); |
|
763 |
} |
|
764 |
#endif |
|
765 |
} |
|
766 |
||
767 |
//------------------------------follow_use_defs--------------------------- |
|
768 |
// Extend the packset by visiting operand definitions of nodes in pack p |
|
769 |
bool SuperWord::follow_use_defs(Node_List* p) { |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
770 |
assert(p->size() == 2, "just checking"); |
1 | 771 |
Node* s1 = p->at(0); |
772 |
Node* s2 = p->at(1); |
|
773 |
assert(s1->req() == s2->req(), "just checking"); |
|
774 |
assert(alignment(s1) + data_size(s1) == alignment(s2), "just checking"); |
|
775 |
||
776 |
if (s1->is_Load()) return false; |
|
777 |
||
778 |
int align = alignment(s1); |
|
779 |
bool changed = false; |
|
780 |
int start = s1->is_Store() ? MemNode::ValueIn : 1; |
|
781 |
int end = s1->is_Store() ? MemNode::ValueIn+1 : s1->req(); |
|
782 |
for (int j = start; j < end; j++) { |
|
783 |
Node* t1 = s1->in(j); |
|
784 |
Node* t2 = s2->in(j); |
|
785 |
if (!in_bb(t1) || !in_bb(t2)) |
|
786 |
continue; |
|
787 |
if (stmts_can_pack(t1, t2, align)) { |
|
788 |
if (est_savings(t1, t2) >= 0) { |
|
789 |
Node_List* pair = new Node_List(); |
|
790 |
pair->push(t1); |
|
791 |
pair->push(t2); |
|
792 |
_packset.append(pair); |
|
793 |
set_alignment(t1, t2, align); |
|
794 |
changed = true; |
|
795 |
} |
|
796 |
} |
|
797 |
} |
|
798 |
return changed; |
|
799 |
} |
|
800 |
||
801 |
//------------------------------follow_def_uses--------------------------- |
|
802 |
// Extend the packset by visiting uses of nodes in pack p |
|
803 |
bool SuperWord::follow_def_uses(Node_List* p) { |
|
804 |
bool changed = false; |
|
805 |
Node* s1 = p->at(0); |
|
806 |
Node* s2 = p->at(1); |
|
807 |
assert(p->size() == 2, "just checking"); |
|
808 |
assert(s1->req() == s2->req(), "just checking"); |
|
809 |
assert(alignment(s1) + data_size(s1) == alignment(s2), "just checking"); |
|
810 |
||
811 |
if (s1->is_Store()) return false; |
|
812 |
||
813 |
int align = alignment(s1); |
|
814 |
int savings = -1; |
|
815 |
Node* u1 = NULL; |
|
816 |
Node* u2 = NULL; |
|
817 |
for (DUIterator_Fast imax, i = s1->fast_outs(imax); i < imax; i++) { |
|
818 |
Node* t1 = s1->fast_out(i); |
|
819 |
if (!in_bb(t1)) continue; |
|
820 |
for (DUIterator_Fast jmax, j = s2->fast_outs(jmax); j < jmax; j++) { |
|
821 |
Node* t2 = s2->fast_out(j); |
|
822 |
if (!in_bb(t2)) continue; |
|
823 |
if (!opnd_positions_match(s1, t1, s2, t2)) |
|
824 |
continue; |
|
825 |
if (stmts_can_pack(t1, t2, align)) { |
|
826 |
int my_savings = est_savings(t1, t2); |
|
827 |
if (my_savings > savings) { |
|
828 |
savings = my_savings; |
|
829 |
u1 = t1; |
|
830 |
u2 = t2; |
|
831 |
} |
|
832 |
} |
|
833 |
} |
|
834 |
} |
|
835 |
if (savings >= 0) { |
|
836 |
Node_List* pair = new Node_List(); |
|
837 |
pair->push(u1); |
|
838 |
pair->push(u2); |
|
839 |
_packset.append(pair); |
|
840 |
set_alignment(u1, u2, align); |
|
841 |
changed = true; |
|
842 |
} |
|
843 |
return changed; |
|
844 |
} |
|
845 |
||
846 |
//---------------------------opnd_positions_match------------------------- |
|
847 |
// Is the use of d1 in u1 at the same operand position as d2 in u2? |
|
848 |
bool SuperWord::opnd_positions_match(Node* d1, Node* u1, Node* d2, Node* u2) { |
|
849 |
uint ct = u1->req(); |
|
850 |
if (ct != u2->req()) return false; |
|
851 |
uint i1 = 0; |
|
852 |
uint i2 = 0; |
|
853 |
do { |
|
854 |
for (i1++; i1 < ct; i1++) if (u1->in(i1) == d1) break; |
|
855 |
for (i2++; i2 < ct; i2++) if (u2->in(i2) == d2) break; |
|
856 |
if (i1 != i2) { |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
857 |
if ((i1 == (3-i2)) && (u2->is_Add() || u2->is_Mul())) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
858 |
// Further analysis relies on operands position matching. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
859 |
u2->swap_edges(i1, i2); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
860 |
} else { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
861 |
return false; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
862 |
} |
1 | 863 |
} |
864 |
} while (i1 < ct); |
|
865 |
return true; |
|
866 |
} |
|
867 |
||
868 |
//------------------------------est_savings--------------------------- |
|
869 |
// Estimate the savings from executing s1 and s2 as a pack |
|
870 |
int SuperWord::est_savings(Node* s1, Node* s2) { |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
871 |
int save_in = 2 - 1; // 2 operations per instruction in packed form |
1 | 872 |
|
873 |
// inputs |
|
874 |
for (uint i = 1; i < s1->req(); i++) { |
|
875 |
Node* x1 = s1->in(i); |
|
876 |
Node* x2 = s2->in(i); |
|
877 |
if (x1 != x2) { |
|
878 |
if (are_adjacent_refs(x1, x2)) { |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
879 |
save_in += adjacent_profit(x1, x2); |
1 | 880 |
} else if (!in_packset(x1, x2)) { |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
881 |
save_in -= pack_cost(2); |
1 | 882 |
} else { |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
883 |
save_in += unpack_cost(2); |
1 | 884 |
} |
885 |
} |
|
886 |
} |
|
887 |
||
888 |
// uses of result |
|
889 |
uint ct = 0; |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
890 |
int save_use = 0; |
1 | 891 |
for (DUIterator_Fast imax, i = s1->fast_outs(imax); i < imax; i++) { |
892 |
Node* s1_use = s1->fast_out(i); |
|
893 |
for (int j = 0; j < _packset.length(); j++) { |
|
894 |
Node_List* p = _packset.at(j); |
|
895 |
if (p->at(0) == s1_use) { |
|
896 |
for (DUIterator_Fast kmax, k = s2->fast_outs(kmax); k < kmax; k++) { |
|
897 |
Node* s2_use = s2->fast_out(k); |
|
898 |
if (p->at(p->size()-1) == s2_use) { |
|
899 |
ct++; |
|
900 |
if (are_adjacent_refs(s1_use, s2_use)) { |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
901 |
save_use += adjacent_profit(s1_use, s2_use); |
1 | 902 |
} |
903 |
} |
|
904 |
} |
|
905 |
} |
|
906 |
} |
|
907 |
} |
|
908 |
||
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
909 |
if (ct < s1->outcnt()) save_use += unpack_cost(1); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
910 |
if (ct < s2->outcnt()) save_use += unpack_cost(1); |
1 | 911 |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
912 |
return MAX2(save_in, save_use); |
1 | 913 |
} |
914 |
||
915 |
//------------------------------costs--------------------------- |
|
916 |
int SuperWord::adjacent_profit(Node* s1, Node* s2) { return 2; } |
|
917 |
int SuperWord::pack_cost(int ct) { return ct; } |
|
918 |
int SuperWord::unpack_cost(int ct) { return ct; } |
|
919 |
||
920 |
//------------------------------combine_packs--------------------------- |
|
921 |
// Combine packs A and B with A.last == B.first into A.first..,A.last,B.second,..B.last |
|
922 |
void SuperWord::combine_packs() { |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
923 |
bool changed = true; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
924 |
// Combine packs regardless max vector size. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
925 |
while (changed) { |
1 | 926 |
changed = false; |
927 |
for (int i = 0; i < _packset.length(); i++) { |
|
928 |
Node_List* p1 = _packset.at(i); |
|
929 |
if (p1 == NULL) continue; |
|
930 |
for (int j = 0; j < _packset.length(); j++) { |
|
931 |
Node_List* p2 = _packset.at(j); |
|
932 |
if (p2 == NULL) continue; |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
933 |
if (i == j) continue; |
1 | 934 |
if (p1->at(p1->size()-1) == p2->at(0)) { |
935 |
for (uint k = 1; k < p2->size(); k++) { |
|
936 |
p1->push(p2->at(k)); |
|
937 |
} |
|
938 |
_packset.at_put(j, NULL); |
|
939 |
changed = true; |
|
940 |
} |
|
941 |
} |
|
942 |
} |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
943 |
} |
1 | 944 |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
945 |
// Split packs which have size greater then max vector size. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
946 |
for (int i = 0; i < _packset.length(); i++) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
947 |
Node_List* p1 = _packset.at(i); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
948 |
if (p1 != NULL) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
949 |
BasicType bt = velt_basic_type(p1->at(0)); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
950 |
uint max_vlen = Matcher::max_vector_size(bt); // Max elements in vector |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
951 |
assert(is_power_of_2(max_vlen), "sanity"); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
952 |
uint psize = p1->size(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
953 |
if (!is_power_of_2(psize)) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
954 |
// Skip pack which can't be vector. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
955 |
// case1: for(...) { a[i] = i; } elements values are different (i+x) |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
956 |
// case2: for(...) { a[i] = b[i+1]; } can't align both, load and store |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
957 |
_packset.at_put(i, NULL); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
958 |
continue; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
959 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
960 |
if (psize > max_vlen) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
961 |
Node_List* pack = new Node_List(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
962 |
for (uint j = 0; j < psize; j++) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
963 |
pack->push(p1->at(j)); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
964 |
if (pack->size() >= max_vlen) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
965 |
assert(is_power_of_2(pack->size()), "sanity"); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
966 |
_packset.append(pack); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
967 |
pack = new Node_List(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
968 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
969 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
970 |
_packset.at_put(i, NULL); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
971 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
972 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
973 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
974 |
|
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
975 |
// Compress list. |
1 | 976 |
for (int i = _packset.length() - 1; i >= 0; i--) { |
977 |
Node_List* p1 = _packset.at(i); |
|
978 |
if (p1 == NULL) { |
|
979 |
_packset.remove_at(i); |
|
980 |
} |
|
981 |
} |
|
982 |
||
983 |
#ifndef PRODUCT |
|
984 |
if (TraceSuperWord) { |
|
985 |
tty->print_cr("\nAfter combine_packs"); |
|
986 |
print_packset(); |
|
987 |
} |
|
988 |
#endif |
|
989 |
} |
|
990 |
||
991 |
//-----------------------------construct_my_pack_map-------------------------- |
|
992 |
// Construct the map from nodes to packs. Only valid after the |
|
993 |
// point where a node is only in one pack (after combine_packs). |
|
994 |
void SuperWord::construct_my_pack_map() { |
|
995 |
Node_List* rslt = NULL; |
|
996 |
for (int i = 0; i < _packset.length(); i++) { |
|
997 |
Node_List* p = _packset.at(i); |
|
998 |
for (uint j = 0; j < p->size(); j++) { |
|
999 |
Node* s = p->at(j); |
|
1000 |
assert(my_pack(s) == NULL, "only in one pack"); |
|
1001 |
set_my_pack(s, p); |
|
1002 |
} |
|
1003 |
} |
|
1004 |
} |
|
1005 |
||
1006 |
//------------------------------filter_packs--------------------------- |
|
1007 |
// Remove packs that are not implemented or not profitable. |
|
1008 |
void SuperWord::filter_packs() { |
|
1009 |
||
1010 |
// Remove packs that are not implemented |
|
1011 |
for (int i = _packset.length() - 1; i >= 0; i--) { |
|
1012 |
Node_List* pk = _packset.at(i); |
|
1013 |
bool impl = implemented(pk); |
|
1014 |
if (!impl) { |
|
1015 |
#ifndef PRODUCT |
|
1016 |
if (TraceSuperWord && Verbose) { |
|
1017 |
tty->print_cr("Unimplemented"); |
|
1018 |
pk->at(0)->dump(); |
|
1019 |
} |
|
1020 |
#endif |
|
1021 |
remove_pack_at(i); |
|
1022 |
} |
|
1023 |
} |
|
1024 |
||
1025 |
// Remove packs that are not profitable |
|
1026 |
bool changed; |
|
1027 |
do { |
|
1028 |
changed = false; |
|
1029 |
for (int i = _packset.length() - 1; i >= 0; i--) { |
|
1030 |
Node_List* pk = _packset.at(i); |
|
1031 |
bool prof = profitable(pk); |
|
1032 |
if (!prof) { |
|
1033 |
#ifndef PRODUCT |
|
1034 |
if (TraceSuperWord && Verbose) { |
|
1035 |
tty->print_cr("Unprofitable"); |
|
1036 |
pk->at(0)->dump(); |
|
1037 |
} |
|
1038 |
#endif |
|
1039 |
remove_pack_at(i); |
|
1040 |
changed = true; |
|
1041 |
} |
|
1042 |
} |
|
1043 |
} while (changed); |
|
1044 |
||
1045 |
#ifndef PRODUCT |
|
1046 |
if (TraceSuperWord) { |
|
1047 |
tty->print_cr("\nAfter filter_packs"); |
|
1048 |
print_packset(); |
|
1049 |
tty->cr(); |
|
1050 |
} |
|
1051 |
#endif |
|
1052 |
} |
|
1053 |
||
1054 |
//------------------------------implemented--------------------------- |
|
1055 |
// Can code be generated for pack p? |
|
1056 |
bool SuperWord::implemented(Node_List* p) { |
|
1057 |
Node* p0 = p->at(0); |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1058 |
return VectorNode::implemented(p0->Opcode(), p->size(), velt_basic_type(p0)); |
1 | 1059 |
} |
1060 |
||
1061 |
//------------------------------profitable--------------------------- |
|
1062 |
// For pack p, are all operands and all uses (with in the block) vector? |
|
1063 |
bool SuperWord::profitable(Node_List* p) { |
|
1064 |
Node* p0 = p->at(0); |
|
1065 |
uint start, end; |
|
1066 |
vector_opd_range(p0, &start, &end); |
|
1067 |
||
1068 |
// Return false if some input is not vector and inside block |
|
1069 |
for (uint i = start; i < end; i++) { |
|
1070 |
if (!is_vector_use(p0, i)) { |
|
1071 |
// For now, return false if not scalar promotion case (inputs are the same.) |
|
2131 | 1072 |
// Later, implement PackNode and allow differing, non-vector inputs |
1 | 1073 |
// (maybe just the ones from outside the block.) |
1074 |
Node* p0_def = p0->in(i); |
|
1075 |
for (uint j = 1; j < p->size(); j++) { |
|
1076 |
Node* use = p->at(j); |
|
1077 |
Node* def = use->in(i); |
|
1078 |
if (p0_def != def) |
|
1079 |
return false; |
|
1080 |
} |
|
1081 |
} |
|
1082 |
} |
|
1083 |
if (!p0->is_Store()) { |
|
1084 |
// For now, return false if not all uses are vector. |
|
1085 |
// Later, implement ExtractNode and allow non-vector uses (maybe |
|
1086 |
// just the ones outside the block.) |
|
1087 |
for (uint i = 0; i < p->size(); i++) { |
|
1088 |
Node* def = p->at(i); |
|
1089 |
for (DUIterator_Fast jmax, j = def->fast_outs(jmax); j < jmax; j++) { |
|
1090 |
Node* use = def->fast_out(j); |
|
1091 |
for (uint k = 0; k < use->req(); k++) { |
|
1092 |
Node* n = use->in(k); |
|
1093 |
if (def == n) { |
|
1094 |
if (!is_vector_use(use, k)) { |
|
1095 |
return false; |
|
1096 |
} |
|
1097 |
} |
|
1098 |
} |
|
1099 |
} |
|
1100 |
} |
|
1101 |
} |
|
1102 |
return true; |
|
1103 |
} |
|
1104 |
||
1105 |
//------------------------------schedule--------------------------- |
|
1106 |
// Adjust the memory graph for the packed operations |
|
1107 |
void SuperWord::schedule() { |
|
1108 |
||
1109 |
// Co-locate in the memory graph the members of each memory pack |
|
1110 |
for (int i = 0; i < _packset.length(); i++) { |
|
1111 |
co_locate_pack(_packset.at(i)); |
|
1112 |
} |
|
1113 |
} |
|
1114 |
||
2334 | 1115 |
//-------------------------------remove_and_insert------------------- |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1116 |
// Remove "current" from its current position in the memory graph and insert |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1117 |
// it after the appropriate insertion point (lip or uip). |
2334 | 1118 |
void SuperWord::remove_and_insert(MemNode *current, MemNode *prev, MemNode *lip, |
1119 |
Node *uip, Unique_Node_List &sched_before) { |
|
1120 |
Node* my_mem = current->in(MemNode::Memory); |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1121 |
bool sched_up = sched_before.member(current); |
2334 | 1122 |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1123 |
// remove current_store from its current position in the memmory graph |
2334 | 1124 |
for (DUIterator i = current->outs(); current->has_out(i); i++) { |
1125 |
Node* use = current->out(i); |
|
1126 |
if (use->is_Mem()) { |
|
1127 |
assert(use->in(MemNode::Memory) == current, "must be"); |
|
1128 |
if (use == prev) { // connect prev to my_mem |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1129 |
_igvn.replace_input_of(use, MemNode::Memory, my_mem); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1130 |
--i; //deleted this edge; rescan position |
2334 | 1131 |
} else if (sched_before.member(use)) { |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1132 |
if (!sched_up) { // Will be moved together with current |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1133 |
_igvn.replace_input_of(use, MemNode::Memory, uip); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1134 |
--i; //deleted this edge; rescan position |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1135 |
} |
2334 | 1136 |
} else { |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1137 |
if (sched_up) { // Will be moved together with current |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1138 |
_igvn.replace_input_of(use, MemNode::Memory, lip); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1139 |
--i; //deleted this edge; rescan position |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1140 |
} |
2334 | 1141 |
} |
1142 |
} |
|
1143 |
} |
|
1144 |
||
1145 |
Node *insert_pt = sched_up ? uip : lip; |
|
1146 |
||
1147 |
// all uses of insert_pt's memory state should use current's instead |
|
1148 |
for (DUIterator i = insert_pt->outs(); insert_pt->has_out(i); i++) { |
|
1149 |
Node* use = insert_pt->out(i); |
|
1150 |
if (use->is_Mem()) { |
|
1151 |
assert(use->in(MemNode::Memory) == insert_pt, "must be"); |
|
12958
009b6c9586d8
7173340: C2: code cleanup: use PhaseIterGVN::replace_edge(Node*, int, Node*) where applicable
kvn
parents:
12594
diff
changeset
|
1152 |
_igvn.replace_input_of(use, MemNode::Memory, current); |
2334 | 1153 |
--i; //deleted this edge; rescan position |
1154 |
} else if (!sched_up && use->is_Phi() && use->bottom_type() == Type::MEMORY) { |
|
1155 |
uint pos; //lip (lower insert point) must be the last one in the memory slice |
|
1156 |
for (pos=1; pos < use->req(); pos++) { |
|
1157 |
if (use->in(pos) == insert_pt) break; |
|
1158 |
} |
|
12958
009b6c9586d8
7173340: C2: code cleanup: use PhaseIterGVN::replace_edge(Node*, int, Node*) where applicable
kvn
parents:
12594
diff
changeset
|
1159 |
_igvn.replace_input_of(use, pos, current); |
2334 | 1160 |
--i; |
1161 |
} |
|
1162 |
} |
|
1163 |
||
1164 |
//connect current to insert_pt |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1165 |
_igvn.replace_input_of(current, MemNode::Memory, insert_pt); |
2334 | 1166 |
} |
1167 |
||
1168 |
//------------------------------co_locate_pack---------------------------------- |
|
1169 |
// To schedule a store pack, we need to move any sandwiched memory ops either before |
|
1170 |
// or after the pack, based upon dependence information: |
|
1171 |
// (1) If any store in the pack depends on the sandwiched memory op, the |
|
1172 |
// sandwiched memory op must be scheduled BEFORE the pack; |
|
1173 |
// (2) If a sandwiched memory op depends on any store in the pack, the |
|
1174 |
// sandwiched memory op must be scheduled AFTER the pack; |
|
1175 |
// (3) If a sandwiched memory op (say, memA) depends on another sandwiched |
|
1176 |
// memory op (say memB), memB must be scheduled before memA. So, if memA is |
|
1177 |
// scheduled before the pack, memB must also be scheduled before the pack; |
|
1178 |
// (4) If there is no dependence restriction for a sandwiched memory op, we simply |
|
1179 |
// schedule this store AFTER the pack |
|
1180 |
// (5) We know there is no dependence cycle, so there in no other case; |
|
1181 |
// (6) Finally, all memory ops in another single pack should be moved in the same direction. |
|
1182 |
// |
|
3799 | 1183 |
// To schedule a load pack, we use the memory state of either the first or the last load in |
1184 |
// the pack, based on the dependence constraint. |
|
1 | 1185 |
void SuperWord::co_locate_pack(Node_List* pk) { |
1186 |
if (pk->at(0)->is_Store()) { |
|
1187 |
MemNode* first = executed_first(pk)->as_Mem(); |
|
1188 |
MemNode* last = executed_last(pk)->as_Mem(); |
|
2334 | 1189 |
Unique_Node_List schedule_before_pack; |
1190 |
Unique_Node_List memops; |
|
1191 |
||
1 | 1192 |
MemNode* current = last->in(MemNode::Memory)->as_Mem(); |
2334 | 1193 |
MemNode* previous = last; |
1 | 1194 |
while (true) { |
1195 |
assert(in_bb(current), "stay in block"); |
|
2334 | 1196 |
memops.push(previous); |
1197 |
for (DUIterator i = current->outs(); current->has_out(i); i++) { |
|
1198 |
Node* use = current->out(i); |
|
1199 |
if (use->is_Mem() && use != previous) |
|
1200 |
memops.push(use); |
|
1201 |
} |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1202 |
if (current == first) break; |
2334 | 1203 |
previous = current; |
1204 |
current = current->in(MemNode::Memory)->as_Mem(); |
|
1205 |
} |
|
1206 |
||
1207 |
// determine which memory operations should be scheduled before the pack |
|
1208 |
for (uint i = 1; i < memops.size(); i++) { |
|
1209 |
Node *s1 = memops.at(i); |
|
1210 |
if (!in_pack(s1, pk) && !schedule_before_pack.member(s1)) { |
|
1211 |
for (uint j = 0; j< i; j++) { |
|
1212 |
Node *s2 = memops.at(j); |
|
1213 |
if (!independent(s1, s2)) { |
|
1214 |
if (in_pack(s2, pk) || schedule_before_pack.member(s2)) { |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1215 |
schedule_before_pack.push(s1); // s1 must be scheduled before |
2334 | 1216 |
Node_List* mem_pk = my_pack(s1); |
1217 |
if (mem_pk != NULL) { |
|
1218 |
for (uint ii = 0; ii < mem_pk->size(); ii++) { |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1219 |
Node* s = mem_pk->at(ii); // follow partner |
2334 | 1220 |
if (memops.member(s) && !schedule_before_pack.member(s)) |
1221 |
schedule_before_pack.push(s); |
|
1222 |
} |
|
1223 |
} |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1224 |
break; |
2334 | 1225 |
} |
1226 |
} |
|
1227 |
} |
|
1228 |
} |
|
1229 |
} |
|
1230 |
||
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1231 |
Node* upper_insert_pt = first->in(MemNode::Memory); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1232 |
// Following code moves loads connected to upper_insert_pt below aliased stores. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1233 |
// Collect such loads here and reconnect them back to upper_insert_pt later. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1234 |
memops.clear(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1235 |
for (DUIterator i = upper_insert_pt->outs(); upper_insert_pt->has_out(i); i++) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1236 |
Node* use = upper_insert_pt->out(i); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1237 |
if (!use->is_Store()) |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1238 |
memops.push(use); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1239 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1240 |
|
2334 | 1241 |
MemNode* lower_insert_pt = last; |
1242 |
previous = last; //previous store in pk |
|
1243 |
current = last->in(MemNode::Memory)->as_Mem(); |
|
1244 |
||
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1245 |
// start scheduling from "last" to "first" |
2334 | 1246 |
while (true) { |
1247 |
assert(in_bb(current), "stay in block"); |
|
1248 |
assert(in_pack(previous, pk), "previous stays in pack"); |
|
1 | 1249 |
Node* my_mem = current->in(MemNode::Memory); |
2334 | 1250 |
|
1 | 1251 |
if (in_pack(current, pk)) { |
2334 | 1252 |
// Forward users of my memory state (except "previous) to my input memory state |
1 | 1253 |
for (DUIterator i = current->outs(); current->has_out(i); i++) { |
1254 |
Node* use = current->out(i); |
|
2334 | 1255 |
if (use->is_Mem() && use != previous) { |
1 | 1256 |
assert(use->in(MemNode::Memory) == current, "must be"); |
2334 | 1257 |
if (schedule_before_pack.member(use)) { |
12958
009b6c9586d8
7173340: C2: code cleanup: use PhaseIterGVN::replace_edge(Node*, int, Node*) where applicable
kvn
parents:
12594
diff
changeset
|
1258 |
_igvn.replace_input_of(use, MemNode::Memory, upper_insert_pt); |
2334 | 1259 |
} else { |
12958
009b6c9586d8
7173340: C2: code cleanup: use PhaseIterGVN::replace_edge(Node*, int, Node*) where applicable
kvn
parents:
12594
diff
changeset
|
1260 |
_igvn.replace_input_of(use, MemNode::Memory, lower_insert_pt); |
2334 | 1261 |
} |
1 | 1262 |
--i; // deleted this edge; rescan position |
1263 |
} |
|
1264 |
} |
|
2334 | 1265 |
previous = current; |
1266 |
} else { // !in_pack(current, pk) ==> a sandwiched store |
|
1267 |
remove_and_insert(current, previous, lower_insert_pt, upper_insert_pt, schedule_before_pack); |
|
1 | 1268 |
} |
2334 | 1269 |
|
1 | 1270 |
if (current == first) break; |
1271 |
current = my_mem->as_Mem(); |
|
2334 | 1272 |
} // end while |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1273 |
|
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1274 |
// Reconnect loads back to upper_insert_pt. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1275 |
for (uint i = 0; i < memops.size(); i++) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1276 |
Node *ld = memops.at(i); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1277 |
if (ld->in(MemNode::Memory) != upper_insert_pt) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1278 |
_igvn.replace_input_of(ld, MemNode::Memory, upper_insert_pt); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1279 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1280 |
} |
2334 | 1281 |
} else if (pk->at(0)->is_Load()) { //load |
3799 | 1282 |
// all loads in the pack should have the same memory state. By default, |
1283 |
// we use the memory state of the last load. However, if any load could |
|
1284 |
// not be moved down due to the dependence constraint, we use the memory |
|
1285 |
// state of the first load. |
|
1286 |
Node* last_mem = executed_last(pk)->in(MemNode::Memory); |
|
1287 |
Node* first_mem = executed_first(pk)->in(MemNode::Memory); |
|
1288 |
bool schedule_last = true; |
|
1289 |
for (uint i = 0; i < pk->size(); i++) { |
|
1290 |
Node* ld = pk->at(i); |
|
1291 |
for (Node* current = last_mem; current != ld->in(MemNode::Memory); |
|
1292 |
current=current->in(MemNode::Memory)) { |
|
1293 |
assert(current != first_mem, "corrupted memory graph"); |
|
1294 |
if(current->is_Mem() && !independent(current, ld)){ |
|
1295 |
schedule_last = false; // a later store depends on this load |
|
1296 |
break; |
|
1297 |
} |
|
1298 |
} |
|
1299 |
} |
|
1300 |
||
1301 |
Node* mem_input = schedule_last ? last_mem : first_mem; |
|
1302 |
_igvn.hash_delete(mem_input); |
|
1303 |
// Give each load the same memory state |
|
1 | 1304 |
for (uint i = 0; i < pk->size(); i++) { |
1305 |
LoadNode* ld = pk->at(i)->as_Load(); |
|
12958
009b6c9586d8
7173340: C2: code cleanup: use PhaseIterGVN::replace_edge(Node*, int, Node*) where applicable
kvn
parents:
12594
diff
changeset
|
1306 |
_igvn.replace_input_of(ld, MemNode::Memory, mem_input); |
1 | 1307 |
} |
1308 |
} |
|
1309 |
} |
|
1310 |
||
1311 |
//------------------------------output--------------------------- |
|
1312 |
// Convert packs into vector node operations |
|
1313 |
void SuperWord::output() { |
|
1314 |
if (_packset.length() == 0) return; |
|
1315 |
||
9101 | 1316 |
#ifndef PRODUCT |
1317 |
if (TraceLoopOpts) { |
|
1318 |
tty->print("SuperWord "); |
|
1319 |
lpt()->dump_head(); |
|
1320 |
} |
|
1321 |
#endif |
|
1322 |
||
1 | 1323 |
// MUST ENSURE main loop's initial value is properly aligned: |
1324 |
// (iv_initial_value + min_iv_offset) % vector_width_in_bytes() == 0 |
|
1325 |
||
1326 |
align_initial_loop_index(align_to_ref()); |
|
1327 |
||
1328 |
// Insert extract (unpack) operations for scalar uses |
|
1329 |
for (int i = 0; i < _packset.length(); i++) { |
|
1330 |
insert_extracts(_packset.at(i)); |
|
1331 |
} |
|
1332 |
||
1333 |
for (int i = 0; i < _block.length(); i++) { |
|
1334 |
Node* n = _block.at(i); |
|
1335 |
Node_List* p = my_pack(n); |
|
1336 |
if (p && n == executed_last(p)) { |
|
1337 |
uint vlen = p->size(); |
|
1338 |
Node* vn = NULL; |
|
1339 |
Node* low_adr = p->at(0); |
|
1340 |
Node* first = executed_first(p); |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1341 |
int opc = n->Opcode(); |
1 | 1342 |
if (n->is_Load()) { |
1343 |
Node* ctl = n->in(MemNode::Control); |
|
1344 |
Node* mem = first->in(MemNode::Memory); |
|
1345 |
Node* adr = low_adr->in(MemNode::Address); |
|
1346 |
const TypePtr* atyp = n->adr_type(); |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1347 |
vn = LoadVectorNode::make(_phase->C, opc, ctl, mem, adr, atyp, vlen, velt_basic_type(n)); |
1 | 1348 |
} else if (n->is_Store()) { |
1349 |
// Promote value to be stored to vector |
|
10255 | 1350 |
Node* val = vector_opd(p, MemNode::ValueIn); |
1 | 1351 |
Node* ctl = n->in(MemNode::Control); |
1352 |
Node* mem = first->in(MemNode::Memory); |
|
1353 |
Node* adr = low_adr->in(MemNode::Address); |
|
1354 |
const TypePtr* atyp = n->adr_type(); |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1355 |
vn = StoreVectorNode::make(_phase->C, opc, ctl, mem, adr, atyp, val, vlen); |
1 | 1356 |
} else if (n->req() == 3) { |
1357 |
// Promote operands to vector |
|
1358 |
Node* in1 = vector_opd(p, 1); |
|
1359 |
Node* in2 = vector_opd(p, 2); |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1360 |
vn = VectorNode::make(_phase->C, opc, in1, in2, vlen, velt_basic_type(n)); |
1 | 1361 |
} else { |
1362 |
ShouldNotReachHere(); |
|
1363 |
} |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1364 |
assert(vn != NULL, "sanity"); |
1 | 1365 |
_phase->_igvn.register_new_node_with_optimizer(vn); |
1366 |
_phase->set_ctrl(vn, _phase->get_ctrl(p->at(0))); |
|
1367 |
for (uint j = 0; j < p->size(); j++) { |
|
1368 |
Node* pm = p->at(j); |
|
5901
c046f8e9c52b
6677629: PhaseIterGVN::subsume_node() should call hash_delete() and add_users_to_worklist()
kvn
parents:
5708
diff
changeset
|
1369 |
_igvn.replace_node(pm, vn); |
1 | 1370 |
} |
1371 |
_igvn._worklist.push(vn); |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1372 |
#ifdef ASSERT |
13108
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
1373 |
if (TraceNewVectors) { |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1374 |
tty->print("new Vector node: "); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1375 |
vn->dump(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1376 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1377 |
#endif |
1 | 1378 |
} |
1379 |
} |
|
1380 |
} |
|
1381 |
||
1382 |
//------------------------------vector_opd--------------------------- |
|
1383 |
// Create a vector operand for the nodes in pack p for operand: in(opd_idx) |
|
10255 | 1384 |
Node* SuperWord::vector_opd(Node_List* p, int opd_idx) { |
1 | 1385 |
Node* p0 = p->at(0); |
1386 |
uint vlen = p->size(); |
|
1387 |
Node* opd = p0->in(opd_idx); |
|
1388 |
||
1389 |
bool same_opd = true; |
|
1390 |
for (uint i = 1; i < vlen; i++) { |
|
1391 |
Node* pi = p->at(i); |
|
1392 |
Node* in = pi->in(opd_idx); |
|
1393 |
if (opd != in) { |
|
1394 |
same_opd = false; |
|
1395 |
break; |
|
1396 |
} |
|
1397 |
} |
|
1398 |
||
1399 |
if (same_opd) { |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1400 |
if (opd->is_Vector() || opd->is_LoadVector()) { |
10255 | 1401 |
return opd; // input is matching vector |
1 | 1402 |
} |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1403 |
assert(!opd->is_StoreVector(), "such vector is not expected here"); |
12594 | 1404 |
// Convert scalar input to vector with the same number of elements as |
1405 |
// p0's vector. Use p0's type because size of operand's container in |
|
1406 |
// vector should match p0's size regardless operand's size. |
|
1407 |
const Type* p0_t = velt_type(p0); |
|
1408 |
VectorNode* vn = VectorNode::scalar2vector(_phase->C, opd, vlen, p0_t); |
|
1 | 1409 |
|
1410 |
_phase->_igvn.register_new_node_with_optimizer(vn); |
|
1411 |
_phase->set_ctrl(vn, _phase->get_ctrl(opd)); |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1412 |
#ifdef ASSERT |
13108
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
1413 |
if (TraceNewVectors) { |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1414 |
tty->print("new Vector node: "); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1415 |
vn->dump(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1416 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1417 |
#endif |
1 | 1418 |
return vn; |
1419 |
} |
|
1420 |
||
1421 |
// Insert pack operation |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1422 |
BasicType bt = velt_basic_type(p0); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1423 |
PackNode* pk = PackNode::make(_phase->C, opd, vlen, bt); |
12594 | 1424 |
DEBUG_ONLY( const BasicType opd_bt = opd->bottom_type()->basic_type(); ) |
1 | 1425 |
|
1426 |
for (uint i = 1; i < vlen; i++) { |
|
1427 |
Node* pi = p->at(i); |
|
1428 |
Node* in = pi->in(opd_idx); |
|
1429 |
assert(my_pack(in) == NULL, "Should already have been unpacked"); |
|
12594 | 1430 |
assert(opd_bt == in->bottom_type()->basic_type(), "all same type"); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1431 |
pk->add_opd(i, in); |
1 | 1432 |
} |
1433 |
_phase->_igvn.register_new_node_with_optimizer(pk); |
|
1434 |
_phase->set_ctrl(pk, _phase->get_ctrl(opd)); |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1435 |
#ifdef ASSERT |
13108
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
1436 |
if (TraceNewVectors) { |
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
1437 |
tty->print("new Vector node: "); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1438 |
pk->dump(); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1439 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1440 |
#endif |
1 | 1441 |
return pk; |
1442 |
} |
|
1443 |
||
1444 |
//------------------------------insert_extracts--------------------------- |
|
1445 |
// If a use of pack p is not a vector use, then replace the |
|
1446 |
// use with an extract operation. |
|
1447 |
void SuperWord::insert_extracts(Node_List* p) { |
|
1448 |
if (p->at(0)->is_Store()) return; |
|
1449 |
assert(_n_idx_list.is_empty(), "empty (node,index) list"); |
|
1450 |
||
1451 |
// Inspect each use of each pack member. For each use that is |
|
1452 |
// not a vector use, replace the use with an extract operation. |
|
1453 |
||
1454 |
for (uint i = 0; i < p->size(); i++) { |
|
1455 |
Node* def = p->at(i); |
|
1456 |
for (DUIterator_Fast jmax, j = def->fast_outs(jmax); j < jmax; j++) { |
|
1457 |
Node* use = def->fast_out(j); |
|
1458 |
for (uint k = 0; k < use->req(); k++) { |
|
1459 |
Node* n = use->in(k); |
|
1460 |
if (def == n) { |
|
1461 |
if (!is_vector_use(use, k)) { |
|
1462 |
_n_idx_list.push(use, k); |
|
1463 |
} |
|
1464 |
} |
|
1465 |
} |
|
1466 |
} |
|
1467 |
} |
|
1468 |
||
1469 |
while (_n_idx_list.is_nonempty()) { |
|
1470 |
Node* use = _n_idx_list.node(); |
|
1471 |
int idx = _n_idx_list.index(); |
|
1472 |
_n_idx_list.pop(); |
|
1473 |
Node* def = use->in(idx); |
|
1474 |
||
1475 |
// Insert extract operation |
|
1476 |
_igvn.hash_delete(def); |
|
1477 |
int def_pos = alignment(def) / data_size(def); |
|
1478 |
||
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1479 |
Node* ex = ExtractNode::make(_phase->C, def, def_pos, velt_basic_type(def)); |
1 | 1480 |
_phase->_igvn.register_new_node_with_optimizer(ex); |
1481 |
_phase->set_ctrl(ex, _phase->get_ctrl(def)); |
|
12958
009b6c9586d8
7173340: C2: code cleanup: use PhaseIterGVN::replace_edge(Node*, int, Node*) where applicable
kvn
parents:
12594
diff
changeset
|
1482 |
_igvn.replace_input_of(use, idx, ex); |
1 | 1483 |
_igvn._worklist.push(def); |
1484 |
||
1485 |
bb_insert_after(ex, bb_idx(def)); |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1486 |
set_velt_type(ex, velt_type(def)); |
1 | 1487 |
} |
1488 |
} |
|
1489 |
||
1490 |
//------------------------------is_vector_use--------------------------- |
|
1491 |
// Is use->in(u_idx) a vector use? |
|
1492 |
bool SuperWord::is_vector_use(Node* use, int u_idx) { |
|
1493 |
Node_List* u_pk = my_pack(use); |
|
1494 |
if (u_pk == NULL) return false; |
|
1495 |
Node* def = use->in(u_idx); |
|
1496 |
Node_List* d_pk = my_pack(def); |
|
1497 |
if (d_pk == NULL) { |
|
1498 |
// check for scalar promotion |
|
1499 |
Node* n = u_pk->at(0)->in(u_idx); |
|
1500 |
for (uint i = 1; i < u_pk->size(); i++) { |
|
1501 |
if (u_pk->at(i)->in(u_idx) != n) return false; |
|
1502 |
} |
|
1503 |
return true; |
|
1504 |
} |
|
1505 |
if (u_pk->size() != d_pk->size()) |
|
1506 |
return false; |
|
1507 |
for (uint i = 0; i < u_pk->size(); i++) { |
|
1508 |
Node* ui = u_pk->at(i); |
|
1509 |
Node* di = d_pk->at(i); |
|
1510 |
if (ui->in(u_idx) != di || alignment(ui) != alignment(di)) |
|
1511 |
return false; |
|
1512 |
} |
|
1513 |
return true; |
|
1514 |
} |
|
1515 |
||
1516 |
//------------------------------construct_bb--------------------------- |
|
1517 |
// Construct reverse postorder list of block members |
|
1518 |
void SuperWord::construct_bb() { |
|
1519 |
Node* entry = bb(); |
|
1520 |
||
1521 |
assert(_stk.length() == 0, "stk is empty"); |
|
1522 |
assert(_block.length() == 0, "block is empty"); |
|
1523 |
assert(_data_entry.length() == 0, "data_entry is empty"); |
|
1524 |
assert(_mem_slice_head.length() == 0, "mem_slice_head is empty"); |
|
1525 |
assert(_mem_slice_tail.length() == 0, "mem_slice_tail is empty"); |
|
1526 |
||
1527 |
// Find non-control nodes with no inputs from within block, |
|
1528 |
// create a temporary map from node _idx to bb_idx for use |
|
1529 |
// by the visited and post_visited sets, |
|
1530 |
// and count number of nodes in block. |
|
1531 |
int bb_ct = 0; |
|
1532 |
for (uint i = 0; i < lpt()->_body.size(); i++ ) { |
|
1533 |
Node *n = lpt()->_body.at(i); |
|
1534 |
set_bb_idx(n, i); // Create a temporary map |
|
1535 |
if (in_bb(n)) { |
|
1536 |
bb_ct++; |
|
1537 |
if (!n->is_CFG()) { |
|
1538 |
bool found = false; |
|
1539 |
for (uint j = 0; j < n->req(); j++) { |
|
1540 |
Node* def = n->in(j); |
|
1541 |
if (def && in_bb(def)) { |
|
1542 |
found = true; |
|
1543 |
break; |
|
1544 |
} |
|
1545 |
} |
|
1546 |
if (!found) { |
|
1547 |
assert(n != entry, "can't be entry"); |
|
1548 |
_data_entry.push(n); |
|
1549 |
} |
|
1550 |
} |
|
1551 |
} |
|
1552 |
} |
|
1553 |
||
1554 |
// Find memory slices (head and tail) |
|
1555 |
for (DUIterator_Fast imax, i = lp()->fast_outs(imax); i < imax; i++) { |
|
1556 |
Node *n = lp()->fast_out(i); |
|
1557 |
if (in_bb(n) && (n->is_Phi() && n->bottom_type() == Type::MEMORY)) { |
|
1558 |
Node* n_tail = n->in(LoopNode::LoopBackControl); |
|
961
7fb3b13d4205
6726999: nsk/stress/jck12a/jck12a010 assert(n != null,"Bad immediate dominator info.")
kvn
parents:
781
diff
changeset
|
1559 |
if (n_tail != n->in(LoopNode::EntryControl)) { |
7fb3b13d4205
6726999: nsk/stress/jck12a/jck12a010 assert(n != null,"Bad immediate dominator info.")
kvn
parents:
781
diff
changeset
|
1560 |
_mem_slice_head.push(n); |
7fb3b13d4205
6726999: nsk/stress/jck12a/jck12a010 assert(n != null,"Bad immediate dominator info.")
kvn
parents:
781
diff
changeset
|
1561 |
_mem_slice_tail.push(n_tail); |
7fb3b13d4205
6726999: nsk/stress/jck12a/jck12a010 assert(n != null,"Bad immediate dominator info.")
kvn
parents:
781
diff
changeset
|
1562 |
} |
1 | 1563 |
} |
1564 |
} |
|
1565 |
||
1566 |
// Create an RPO list of nodes in block |
|
1567 |
||
1568 |
visited_clear(); |
|
1569 |
post_visited_clear(); |
|
1570 |
||
1571 |
// Push all non-control nodes with no inputs from within block, then control entry |
|
1572 |
for (int j = 0; j < _data_entry.length(); j++) { |
|
1573 |
Node* n = _data_entry.at(j); |
|
1574 |
visited_set(n); |
|
1575 |
_stk.push(n); |
|
1576 |
} |
|
1577 |
visited_set(entry); |
|
1578 |
_stk.push(entry); |
|
1579 |
||
1580 |
// Do a depth first walk over out edges |
|
1581 |
int rpo_idx = bb_ct - 1; |
|
1582 |
int size; |
|
1583 |
while ((size = _stk.length()) > 0) { |
|
1584 |
Node* n = _stk.top(); // Leave node on stack |
|
1585 |
if (!visited_test_set(n)) { |
|
1586 |
// forward arc in graph |
|
1587 |
} else if (!post_visited_test(n)) { |
|
1588 |
// cross or back arc |
|
1589 |
for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { |
|
1590 |
Node *use = n->fast_out(i); |
|
1591 |
if (in_bb(use) && !visited_test(use) && |
|
1592 |
// Don't go around backedge |
|
1593 |
(!use->is_Phi() || n == entry)) { |
|
1594 |
_stk.push(use); |
|
1595 |
} |
|
1596 |
} |
|
1597 |
if (_stk.length() == size) { |
|
1598 |
// There were no additional uses, post visit node now |
|
1599 |
_stk.pop(); // Remove node from stack |
|
1600 |
assert(rpo_idx >= 0, ""); |
|
1601 |
_block.at_put_grow(rpo_idx, n); |
|
1602 |
rpo_idx--; |
|
1603 |
post_visited_set(n); |
|
1604 |
assert(rpo_idx >= 0 || _stk.is_empty(), ""); |
|
1605 |
} |
|
1606 |
} else { |
|
1607 |
_stk.pop(); // Remove post-visited node from stack |
|
1608 |
} |
|
1609 |
} |
|
1610 |
||
1611 |
// Create real map of block indices for nodes |
|
1612 |
for (int j = 0; j < _block.length(); j++) { |
|
1613 |
Node* n = _block.at(j); |
|
1614 |
set_bb_idx(n, j); |
|
1615 |
} |
|
1616 |
||
1617 |
initialize_bb(); // Ensure extra info is allocated. |
|
1618 |
||
1619 |
#ifndef PRODUCT |
|
1620 |
if (TraceSuperWord) { |
|
1621 |
print_bb(); |
|
1622 |
tty->print_cr("\ndata entry nodes: %s", _data_entry.length() > 0 ? "" : "NONE"); |
|
1623 |
for (int m = 0; m < _data_entry.length(); m++) { |
|
1624 |
tty->print("%3d ", m); |
|
1625 |
_data_entry.at(m)->dump(); |
|
1626 |
} |
|
1627 |
tty->print_cr("\nmemory slices: %s", _mem_slice_head.length() > 0 ? "" : "NONE"); |
|
1628 |
for (int m = 0; m < _mem_slice_head.length(); m++) { |
|
1629 |
tty->print("%3d ", m); _mem_slice_head.at(m)->dump(); |
|
1630 |
tty->print(" "); _mem_slice_tail.at(m)->dump(); |
|
1631 |
} |
|
1632 |
} |
|
1633 |
#endif |
|
1634 |
assert(rpo_idx == -1 && bb_ct == _block.length(), "all block members found"); |
|
1635 |
} |
|
1636 |
||
1637 |
//------------------------------initialize_bb--------------------------- |
|
1638 |
// Initialize per node info |
|
1639 |
void SuperWord::initialize_bb() { |
|
1640 |
Node* last = _block.at(_block.length() - 1); |
|
1641 |
grow_node_info(bb_idx(last)); |
|
1642 |
} |
|
1643 |
||
1644 |
//------------------------------bb_insert_after--------------------------- |
|
1645 |
// Insert n into block after pos |
|
1646 |
void SuperWord::bb_insert_after(Node* n, int pos) { |
|
1647 |
int n_pos = pos + 1; |
|
1648 |
// Make room |
|
1649 |
for (int i = _block.length() - 1; i >= n_pos; i--) { |
|
1650 |
_block.at_put_grow(i+1, _block.at(i)); |
|
1651 |
} |
|
1652 |
for (int j = _node_info.length() - 1; j >= n_pos; j--) { |
|
1653 |
_node_info.at_put_grow(j+1, _node_info.at(j)); |
|
1654 |
} |
|
1655 |
// Set value |
|
1656 |
_block.at_put_grow(n_pos, n); |
|
1657 |
_node_info.at_put_grow(n_pos, SWNodeInfo::initial); |
|
1658 |
// Adjust map from node->_idx to _block index |
|
1659 |
for (int i = n_pos; i < _block.length(); i++) { |
|
1660 |
set_bb_idx(_block.at(i), i); |
|
1661 |
} |
|
1662 |
} |
|
1663 |
||
1664 |
//------------------------------compute_max_depth--------------------------- |
|
1665 |
// Compute max depth for expressions from beginning of block |
|
1666 |
// Use to prune search paths during test for independence. |
|
1667 |
void SuperWord::compute_max_depth() { |
|
1668 |
int ct = 0; |
|
1669 |
bool again; |
|
1670 |
do { |
|
1671 |
again = false; |
|
1672 |
for (int i = 0; i < _block.length(); i++) { |
|
1673 |
Node* n = _block.at(i); |
|
1674 |
if (!n->is_Phi()) { |
|
1675 |
int d_orig = depth(n); |
|
1676 |
int d_in = 0; |
|
1677 |
for (DepPreds preds(n, _dg); !preds.done(); preds.next()) { |
|
1678 |
Node* pred = preds.current(); |
|
1679 |
if (in_bb(pred)) { |
|
1680 |
d_in = MAX2(d_in, depth(pred)); |
|
1681 |
} |
|
1682 |
} |
|
1683 |
if (d_in + 1 != d_orig) { |
|
1684 |
set_depth(n, d_in + 1); |
|
1685 |
again = true; |
|
1686 |
} |
|
1687 |
} |
|
1688 |
} |
|
1689 |
ct++; |
|
1690 |
} while (again); |
|
1691 |
#ifndef PRODUCT |
|
1692 |
if (TraceSuperWord && Verbose) |
|
1693 |
tty->print_cr("compute_max_depth iterated: %d times", ct); |
|
1694 |
#endif |
|
1695 |
} |
|
1696 |
||
1697 |
//-------------------------compute_vector_element_type----------------------- |
|
1698 |
// Compute necessary vector element type for expressions |
|
1699 |
// This propagates backwards a narrower integer type when the |
|
1700 |
// upper bits of the value are not needed. |
|
1701 |
// Example: char a,b,c; a = b + c; |
|
1702 |
// Normally the type of the add is integer, but for packed character |
|
1703 |
// operations the type of the add needs to be char. |
|
1704 |
void SuperWord::compute_vector_element_type() { |
|
1705 |
#ifndef PRODUCT |
|
1706 |
if (TraceSuperWord && Verbose) |
|
1707 |
tty->print_cr("\ncompute_velt_type:"); |
|
1708 |
#endif |
|
1709 |
||
1710 |
// Initial type |
|
1711 |
for (int i = 0; i < _block.length(); i++) { |
|
1712 |
Node* n = _block.at(i); |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1713 |
set_velt_type(n, container_type(n)); |
1 | 1714 |
} |
1715 |
||
1716 |
// Propagate narrowed type backwards through operations |
|
1717 |
// that don't depend on higher order bits |
|
1718 |
for (int i = _block.length() - 1; i >= 0; i--) { |
|
1719 |
Node* n = _block.at(i); |
|
1720 |
// Only integer types need be examined |
|
1721 |
if (n->bottom_type()->isa_int()) { |
|
1722 |
uint start, end; |
|
1723 |
vector_opd_range(n, &start, &end); |
|
1724 |
const Type* vt = velt_type(n); |
|
1725 |
||
1726 |
for (uint j = start; j < end; j++) { |
|
1727 |
Node* in = n->in(j); |
|
1728 |
// Don't propagate through a type conversion |
|
1729 |
if (n->bottom_type() != in->bottom_type()) |
|
1730 |
continue; |
|
1731 |
switch(in->Opcode()) { |
|
1732 |
case Op_AddI: case Op_AddL: |
|
1733 |
case Op_SubI: case Op_SubL: |
|
1734 |
case Op_MulI: case Op_MulL: |
|
1735 |
case Op_AndI: case Op_AndL: |
|
1736 |
case Op_OrI: case Op_OrL: |
|
1737 |
case Op_XorI: case Op_XorL: |
|
1738 |
case Op_LShiftI: case Op_LShiftL: |
|
1739 |
case Op_CMoveI: case Op_CMoveL: |
|
1740 |
if (in_bb(in)) { |
|
1741 |
bool same_type = true; |
|
1742 |
for (DUIterator_Fast kmax, k = in->fast_outs(kmax); k < kmax; k++) { |
|
1743 |
Node *use = in->fast_out(k); |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1744 |
if (!in_bb(use) || !same_velt_type(use, n)) { |
1 | 1745 |
same_type = false; |
1746 |
break; |
|
1747 |
} |
|
1748 |
} |
|
1749 |
if (same_type) { |
|
1750 |
set_velt_type(in, vt); |
|
1751 |
} |
|
1752 |
} |
|
1753 |
} |
|
1754 |
} |
|
1755 |
} |
|
1756 |
} |
|
1757 |
#ifndef PRODUCT |
|
1758 |
if (TraceSuperWord && Verbose) { |
|
1759 |
for (int i = 0; i < _block.length(); i++) { |
|
1760 |
Node* n = _block.at(i); |
|
1761 |
velt_type(n)->dump(); |
|
1762 |
tty->print("\t"); |
|
1763 |
n->dump(); |
|
1764 |
} |
|
1765 |
} |
|
1766 |
#endif |
|
1767 |
} |
|
1768 |
||
1769 |
//------------------------------memory_alignment--------------------------- |
|
1770 |
// Alignment within a vector memory reference |
|
1771 |
int SuperWord::memory_alignment(MemNode* s, int iv_adjust_in_bytes) { |
|
1772 |
SWPointer p(s, this); |
|
1773 |
if (!p.valid()) { |
|
1774 |
return bottom_align; |
|
1775 |
} |
|
13108
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
1776 |
int vw = vector_width_in_bytes(s); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1777 |
if (vw < 2) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1778 |
return bottom_align; // No vectors for this type |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1779 |
} |
1 | 1780 |
int offset = p.offset_in_bytes(); |
1781 |
offset += iv_adjust_in_bytes; |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1782 |
int off_rem = offset % vw; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1783 |
int off_mod = off_rem >= 0 ? off_rem : off_rem + vw; |
1 | 1784 |
return off_mod; |
1785 |
} |
|
1786 |
||
1787 |
//---------------------------container_type--------------------------- |
|
1788 |
// Smallest type containing range of values |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1789 |
const Type* SuperWord::container_type(Node* n) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1790 |
if (n->is_Mem()) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1791 |
return Type::get_const_basic_type(n->as_Mem()->memory_type()); |
1 | 1792 |
} |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1793 |
const Type* t = _igvn.type(n); |
1 | 1794 |
if (t->basic_type() == T_INT) { |
1795 |
if (t->higher_equal(TypeInt::BOOL)) return TypeInt::BOOL; |
|
1796 |
if (t->higher_equal(TypeInt::BYTE)) return TypeInt::BYTE; |
|
1797 |
if (t->higher_equal(TypeInt::CHAR)) return TypeInt::CHAR; |
|
1798 |
if (t->higher_equal(TypeInt::SHORT)) return TypeInt::SHORT; |
|
1799 |
return TypeInt::INT; |
|
1800 |
} |
|
1801 |
return t; |
|
1802 |
} |
|
1803 |
||
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1804 |
bool SuperWord::same_velt_type(Node* n1, Node* n2) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1805 |
const Type* vt1 = velt_type(n1); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1806 |
const Type* vt2 = velt_type(n1); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1807 |
if (vt1->basic_type() == T_INT && vt2->basic_type() == T_INT) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1808 |
// Compare vectors element sizes for integer types. |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1809 |
return data_size(n1) == data_size(n2); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1810 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1811 |
return vt1 == vt2; |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1812 |
} |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1813 |
|
1 | 1814 |
//-------------------------vector_opd_range----------------------- |
1815 |
// (Start, end] half-open range defining which operands are vector |
|
1816 |
void SuperWord::vector_opd_range(Node* n, uint* start, uint* end) { |
|
1817 |
switch (n->Opcode()) { |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1818 |
case Op_LoadB: case Op_LoadUB: |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1819 |
case Op_LoadS: case Op_LoadUS: |
1 | 1820 |
case Op_LoadI: case Op_LoadL: |
1821 |
case Op_LoadF: case Op_LoadD: |
|
1822 |
case Op_LoadP: |
|
1823 |
*start = 0; |
|
1824 |
*end = 0; |
|
1825 |
return; |
|
1826 |
case Op_StoreB: case Op_StoreC: |
|
1827 |
case Op_StoreI: case Op_StoreL: |
|
1828 |
case Op_StoreF: case Op_StoreD: |
|
1829 |
case Op_StoreP: |
|
1830 |
*start = MemNode::ValueIn; |
|
1831 |
*end = *start + 1; |
|
1832 |
return; |
|
1833 |
case Op_LShiftI: case Op_LShiftL: |
|
1834 |
*start = 1; |
|
1835 |
*end = 2; |
|
1836 |
return; |
|
1837 |
case Op_CMoveI: case Op_CMoveL: case Op_CMoveF: case Op_CMoveD: |
|
1838 |
*start = 2; |
|
1839 |
*end = n->req(); |
|
1840 |
return; |
|
1841 |
} |
|
1842 |
*start = 1; |
|
1843 |
*end = n->req(); // default is all operands |
|
1844 |
} |
|
1845 |
||
1846 |
//------------------------------in_packset--------------------------- |
|
1847 |
// Are s1 and s2 in a pack pair and ordered as s1,s2? |
|
1848 |
bool SuperWord::in_packset(Node* s1, Node* s2) { |
|
1849 |
for (int i = 0; i < _packset.length(); i++) { |
|
1850 |
Node_List* p = _packset.at(i); |
|
1851 |
assert(p->size() == 2, "must be"); |
|
1852 |
if (p->at(0) == s1 && p->at(p->size()-1) == s2) { |
|
1853 |
return true; |
|
1854 |
} |
|
1855 |
} |
|
1856 |
return false; |
|
1857 |
} |
|
1858 |
||
1859 |
//------------------------------in_pack--------------------------- |
|
1860 |
// Is s in pack p? |
|
1861 |
Node_List* SuperWord::in_pack(Node* s, Node_List* p) { |
|
1862 |
for (uint i = 0; i < p->size(); i++) { |
|
1863 |
if (p->at(i) == s) { |
|
1864 |
return p; |
|
1865 |
} |
|
1866 |
} |
|
1867 |
return NULL; |
|
1868 |
} |
|
1869 |
||
1870 |
//------------------------------remove_pack_at--------------------------- |
|
1871 |
// Remove the pack at position pos in the packset |
|
1872 |
void SuperWord::remove_pack_at(int pos) { |
|
1873 |
Node_List* p = _packset.at(pos); |
|
1874 |
for (uint i = 0; i < p->size(); i++) { |
|
1875 |
Node* s = p->at(i); |
|
1876 |
set_my_pack(s, NULL); |
|
1877 |
} |
|
1878 |
_packset.remove_at(pos); |
|
1879 |
} |
|
1880 |
||
1881 |
//------------------------------executed_first--------------------------- |
|
1882 |
// Return the node executed first in pack p. Uses the RPO block list |
|
1883 |
// to determine order. |
|
1884 |
Node* SuperWord::executed_first(Node_List* p) { |
|
1885 |
Node* n = p->at(0); |
|
1886 |
int n_rpo = bb_idx(n); |
|
1887 |
for (uint i = 1; i < p->size(); i++) { |
|
1888 |
Node* s = p->at(i); |
|
1889 |
int s_rpo = bb_idx(s); |
|
1890 |
if (s_rpo < n_rpo) { |
|
1891 |
n = s; |
|
1892 |
n_rpo = s_rpo; |
|
1893 |
} |
|
1894 |
} |
|
1895 |
return n; |
|
1896 |
} |
|
1897 |
||
1898 |
//------------------------------executed_last--------------------------- |
|
1899 |
// Return the node executed last in pack p. |
|
1900 |
Node* SuperWord::executed_last(Node_List* p) { |
|
1901 |
Node* n = p->at(0); |
|
1902 |
int n_rpo = bb_idx(n); |
|
1903 |
for (uint i = 1; i < p->size(); i++) { |
|
1904 |
Node* s = p->at(i); |
|
1905 |
int s_rpo = bb_idx(s); |
|
1906 |
if (s_rpo > n_rpo) { |
|
1907 |
n = s; |
|
1908 |
n_rpo = s_rpo; |
|
1909 |
} |
|
1910 |
} |
|
1911 |
return n; |
|
1912 |
} |
|
1913 |
||
1914 |
//----------------------------align_initial_loop_index--------------------------- |
|
1915 |
// Adjust pre-loop limit so that in main loop, a load/store reference |
|
1916 |
// to align_to_ref will be a position zero in the vector. |
|
1917 |
// (iv + k) mod vector_align == 0 |
|
1918 |
void SuperWord::align_initial_loop_index(MemNode* align_to_ref) { |
|
1919 |
CountedLoopNode *main_head = lp()->as_CountedLoop(); |
|
1920 |
assert(main_head->is_main_loop(), ""); |
|
1921 |
CountedLoopEndNode* pre_end = get_pre_loop_end(main_head); |
|
1922 |
assert(pre_end != NULL, ""); |
|
1923 |
Node *pre_opaq1 = pre_end->limit(); |
|
1924 |
assert(pre_opaq1->Opcode() == Op_Opaque1, ""); |
|
1925 |
Opaque1Node *pre_opaq = (Opaque1Node*)pre_opaq1; |
|
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1926 |
Node *lim0 = pre_opaq->in(1); |
1 | 1927 |
|
1928 |
// Where we put new limit calculations |
|
1929 |
Node *pre_ctrl = pre_end->loopnode()->in(LoopNode::EntryControl); |
|
1930 |
||
1931 |
// Ensure the original loop limit is available from the |
|
1932 |
// pre-loop Opaque1 node. |
|
1933 |
Node *orig_limit = pre_opaq->original_loop_limit(); |
|
1934 |
assert(orig_limit != NULL && _igvn.type(orig_limit) != Type::TOP, ""); |
|
1935 |
||
1936 |
SWPointer align_to_ref_p(align_to_ref, this); |
|
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1937 |
assert(align_to_ref_p.valid(), "sanity"); |
1 | 1938 |
|
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1939 |
// Given: |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1940 |
// lim0 == original pre loop limit |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1941 |
// V == v_align (power of 2) |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1942 |
// invar == extra invariant piece of the address expression |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1943 |
// e == k [ +/- invar ] |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1944 |
// |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1945 |
// When reassociating expressions involving '%' the basic rules are: |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1946 |
// (a - b) % k == 0 => a % k == b % k |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1947 |
// and: |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1948 |
// (a + b) % k == 0 => a % k == (k - b) % k |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1949 |
// |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1950 |
// For stride > 0 && scale > 0, |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1951 |
// Derive the new pre-loop limit "lim" such that the two constraints: |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1952 |
// (1) lim = lim0 + N (where N is some positive integer < V) |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1953 |
// (2) (e + lim) % V == 0 |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1954 |
// are true. |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1955 |
// |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1956 |
// Substituting (1) into (2), |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1957 |
// (e + lim0 + N) % V == 0 |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1958 |
// solve for N: |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1959 |
// N = (V - (e + lim0)) % V |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1960 |
// substitute back into (1), so that new limit |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1961 |
// lim = lim0 + (V - (e + lim0)) % V |
1 | 1962 |
// |
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1963 |
// For stride > 0 && scale < 0 |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1964 |
// Constraints: |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1965 |
// lim = lim0 + N |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1966 |
// (e - lim) % V == 0 |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1967 |
// Solving for lim: |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1968 |
// (e - lim0 - N) % V == 0 |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1969 |
// N = (e - lim0) % V |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1970 |
// lim = lim0 + (e - lim0) % V |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1971 |
// |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1972 |
// For stride < 0 && scale > 0 |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1973 |
// Constraints: |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1974 |
// lim = lim0 - N |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1975 |
// (e + lim) % V == 0 |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1976 |
// Solving for lim: |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1977 |
// (e + lim0 - N) % V == 0 |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1978 |
// N = (e + lim0) % V |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1979 |
// lim = lim0 - (e + lim0) % V |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1980 |
// |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1981 |
// For stride < 0 && scale < 0 |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1982 |
// Constraints: |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1983 |
// lim = lim0 - N |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1984 |
// (e - lim) % V == 0 |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1985 |
// Solving for lim: |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1986 |
// (e - lim0 + N) % V == 0 |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1987 |
// N = (V - (e - lim0)) % V |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1988 |
// lim = lim0 - (V - (e - lim0)) % V |
1 | 1989 |
|
13108
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
1990 |
int vw = vector_width_in_bytes(align_to_ref); |
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1991 |
int stride = iv_stride(); |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1992 |
int scale = align_to_ref_p.scale_in_bytes(); |
1 | 1993 |
int elt_size = align_to_ref_p.memory_size(); |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
1994 |
int v_align = vw / elt_size; |
13108
6d27f658925c
7177923: SIGBUS on sparc in compiled code for java.util.Calendar.clear()
kvn
parents:
13104
diff
changeset
|
1995 |
assert(v_align > 1, "sanity"); |
1 | 1996 |
int k = align_to_ref_p.offset_in_bytes() / elt_size; |
1997 |
||
1998 |
Node *kn = _igvn.intcon(k); |
|
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
1999 |
|
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2000 |
Node *e = kn; |
1 | 2001 |
if (align_to_ref_p.invar() != NULL) { |
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2002 |
// incorporate any extra invariant piece producing k +/- invar >>> log2(elt) |
1 | 2003 |
Node* log2_elt = _igvn.intcon(exact_log2(elt_size)); |
2004 |
Node* aref = new (_phase->C, 3) URShiftINode(align_to_ref_p.invar(), log2_elt); |
|
2005 |
_phase->_igvn.register_new_node_with_optimizer(aref); |
|
2006 |
_phase->set_ctrl(aref, pre_ctrl); |
|
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2007 |
if (align_to_ref_p.negate_invar()) { |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2008 |
e = new (_phase->C, 3) SubINode(e, aref); |
1 | 2009 |
} else { |
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2010 |
e = new (_phase->C, 3) AddINode(e, aref); |
1 | 2011 |
} |
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2012 |
_phase->_igvn.register_new_node_with_optimizer(e); |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2013 |
_phase->set_ctrl(e, pre_ctrl); |
1 | 2014 |
} |
13104
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2015 |
if (vw > ObjectAlignmentInBytes) { |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2016 |
// incorporate base e +/- base && Mask >>> log2(elt) |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2017 |
Node* mask = _igvn.MakeConX(~(-1 << exact_log2(vw))); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2018 |
Node* xbase = new(_phase->C, 2) CastP2XNode(NULL, align_to_ref_p.base()); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2019 |
_phase->_igvn.register_new_node_with_optimizer(xbase); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2020 |
Node* masked_xbase = new (_phase->C, 3) AndXNode(xbase, mask); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2021 |
_phase->_igvn.register_new_node_with_optimizer(masked_xbase); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2022 |
#ifdef _LP64 |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2023 |
masked_xbase = new (_phase->C, 2) ConvL2INode(masked_xbase); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2024 |
_phase->_igvn.register_new_node_with_optimizer(masked_xbase); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2025 |
#endif |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2026 |
Node* log2_elt = _igvn.intcon(exact_log2(elt_size)); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2027 |
Node* bref = new (_phase->C, 3) URShiftINode(masked_xbase, log2_elt); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2028 |
_phase->_igvn.register_new_node_with_optimizer(bref); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2029 |
_phase->set_ctrl(bref, pre_ctrl); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2030 |
e = new (_phase->C, 3) AddINode(e, bref); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2031 |
_phase->_igvn.register_new_node_with_optimizer(e); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2032 |
_phase->set_ctrl(e, pre_ctrl); |
657b387034fb
7119644: Increase superword's vector size up to 256 bits
kvn
parents:
12958
diff
changeset
|
2033 |
} |
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2034 |
|
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2035 |
// compute e +/- lim0 |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2036 |
if (scale < 0) { |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2037 |
e = new (_phase->C, 3) SubINode(e, lim0); |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2038 |
} else { |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2039 |
e = new (_phase->C, 3) AddINode(e, lim0); |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2040 |
} |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2041 |
_phase->_igvn.register_new_node_with_optimizer(e); |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2042 |
_phase->set_ctrl(e, pre_ctrl); |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2043 |
|
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2044 |
if (stride * scale > 0) { |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2045 |
// compute V - (e +/- lim0) |
1 | 2046 |
Node* va = _igvn.intcon(v_align); |
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2047 |
e = new (_phase->C, 3) SubINode(va, e); |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2048 |
_phase->_igvn.register_new_node_with_optimizer(e); |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2049 |
_phase->set_ctrl(e, pre_ctrl); |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2050 |
} |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2051 |
// compute N = (exp) % V |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2052 |
Node* va_msk = _igvn.intcon(v_align - 1); |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2053 |
Node* N = new (_phase->C, 3) AndINode(e, va_msk); |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2054 |
_phase->_igvn.register_new_node_with_optimizer(N); |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2055 |
_phase->set_ctrl(N, pre_ctrl); |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2056 |
|
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2057 |
// substitute back into (1), so that new limit |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2058 |
// lim = lim0 + N |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2059 |
Node* lim; |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2060 |
if (stride < 0) { |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2061 |
lim = new (_phase->C, 3) SubINode(lim0, N); |
1 | 2062 |
} else { |
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2063 |
lim = new (_phase->C, 3) AddINode(lim0, N); |
1 | 2064 |
} |
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2065 |
_phase->_igvn.register_new_node_with_optimizer(lim); |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2066 |
_phase->set_ctrl(lim, pre_ctrl); |
1 | 2067 |
Node* constrained = |
245
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2068 |
(stride > 0) ? (Node*) new (_phase->C,3) MinINode(lim, orig_limit) |
b9df534a2faa
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
202
diff
changeset
|
2069 |
: (Node*) new (_phase->C,3) MaxINode(lim, orig_limit); |
1 | 2070 |
_phase->_igvn.register_new_node_with_optimizer(constrained); |
2071 |
_phase->set_ctrl(constrained, pre_ctrl); |
|
2072 |
_igvn.hash_delete(pre_opaq); |
|
2073 |
pre_opaq->set_req(1, constrained); |
|
2074 |
} |
|
2075 |
||
2076 |
//----------------------------get_pre_loop_end--------------------------- |
|
2077 |
// Find pre loop end from main loop. Returns null if none. |
|
2078 |
CountedLoopEndNode* SuperWord::get_pre_loop_end(CountedLoopNode *cl) { |
|
2079 |
Node *ctrl = cl->in(LoopNode::EntryControl); |
|
2080 |
if (!ctrl->is_IfTrue() && !ctrl->is_IfFalse()) return NULL; |
|
2081 |
Node *iffm = ctrl->in(0); |
|
2082 |
if (!iffm->is_If()) return NULL; |
|
2083 |
Node *p_f = iffm->in(0); |
|
2084 |
if (!p_f->is_IfFalse()) return NULL; |
|
2085 |
if (!p_f->in(0)->is_CountedLoopEnd()) return NULL; |
|
2086 |
CountedLoopEndNode *pre_end = p_f->in(0)->as_CountedLoopEnd(); |
|
2087 |
if (!pre_end->loopnode()->is_pre_loop()) return NULL; |
|
2088 |
return pre_end; |
|
2089 |
} |
|
2090 |
||
2091 |
||
2092 |
//------------------------------init--------------------------- |
|
2093 |
void SuperWord::init() { |
|
2094 |
_dg.init(); |
|
2095 |
_packset.clear(); |
|
2096 |
_disjoint_ptrs.clear(); |
|
2097 |
_block.clear(); |
|
2098 |
_data_entry.clear(); |
|
2099 |
_mem_slice_head.clear(); |
|
2100 |
_mem_slice_tail.clear(); |
|
2101 |
_node_info.clear(); |
|
2102 |
_align_to_ref = NULL; |
|
2103 |
_lpt = NULL; |
|
2104 |
_lp = NULL; |
|
2105 |
_bb = NULL; |
|
2106 |
_iv = NULL; |
|
2107 |
} |
|
2108 |
||
2109 |
//------------------------------print_packset--------------------------- |
|
2110 |
void SuperWord::print_packset() { |
|
2111 |
#ifndef PRODUCT |
|
2112 |
tty->print_cr("packset"); |
|
2113 |
for (int i = 0; i < _packset.length(); i++) { |
|
2114 |
tty->print_cr("Pack: %d", i); |
|
2115 |
Node_List* p = _packset.at(i); |
|
2116 |
print_pack(p); |
|
2117 |
} |
|
2118 |
#endif |
|
2119 |
} |
|
2120 |
||
2121 |
//------------------------------print_pack--------------------------- |
|
2122 |
void SuperWord::print_pack(Node_List* p) { |
|
2123 |
for (uint i = 0; i < p->size(); i++) { |
|
2124 |
print_stmt(p->at(i)); |
|
2125 |
} |
|
2126 |
} |
|
2127 |
||
2128 |
//------------------------------print_bb--------------------------- |
|
2129 |
void SuperWord::print_bb() { |
|
2130 |
#ifndef PRODUCT |
|
2131 |
tty->print_cr("\nBlock"); |
|
2132 |
for (int i = 0; i < _block.length(); i++) { |
|
2133 |
Node* n = _block.at(i); |
|
2134 |
tty->print("%d ", i); |
|
2135 |
if (n) { |
|
2136 |
n->dump(); |
|
2137 |
} |
|
2138 |
} |
|
2139 |
#endif |
|
2140 |
} |
|
2141 |
||
2142 |
//------------------------------print_stmt--------------------------- |
|
2143 |
void SuperWord::print_stmt(Node* s) { |
|
2144 |
#ifndef PRODUCT |
|
2145 |
tty->print(" align: %d \t", alignment(s)); |
|
2146 |
s->dump(); |
|
2147 |
#endif |
|
2148 |
} |
|
2149 |
||
2150 |
//------------------------------blank--------------------------- |
|
2151 |
char* SuperWord::blank(uint depth) { |
|
2152 |
static char blanks[101]; |
|
2153 |
assert(depth < 101, "too deep"); |
|
2154 |
for (uint i = 0; i < depth; i++) blanks[i] = ' '; |
|
2155 |
blanks[depth] = '\0'; |
|
2156 |
return blanks; |
|
2157 |
} |
|
2158 |
||
2159 |
||
2160 |
//==============================SWPointer=========================== |
|
2161 |
||
2162 |
//----------------------------SWPointer------------------------ |
|
2163 |
SWPointer::SWPointer(MemNode* mem, SuperWord* slp) : |
|
2164 |
_mem(mem), _slp(slp), _base(NULL), _adr(NULL), |
|
2165 |
_scale(0), _offset(0), _invar(NULL), _negate_invar(false) { |
|
2166 |
||
2167 |
Node* adr = mem->in(MemNode::Address); |
|
2168 |
if (!adr->is_AddP()) { |
|
2169 |
assert(!valid(), "too complex"); |
|
2170 |
return; |
|
2171 |
} |
|
2172 |
// Match AddP(base, AddP(ptr, k*iv [+ invariant]), constant) |
|
2173 |
Node* base = adr->in(AddPNode::Base); |
|
4428
d1617f46285d
6852078: HSX 14/16 in jdk 5.0: api/javax_management api/org_omg jck tests crashes or make tnameserv crash
cfang
parents:
3906
diff
changeset
|
2174 |
//unsafe reference could not be aligned appropriately without runtime checking |
d1617f46285d
6852078: HSX 14/16 in jdk 5.0: api/javax_management api/org_omg jck tests crashes or make tnameserv crash
cfang
parents:
3906
diff
changeset
|
2175 |
if (base == NULL || base->bottom_type() == Type::TOP) { |
d1617f46285d
6852078: HSX 14/16 in jdk 5.0: api/javax_management api/org_omg jck tests crashes or make tnameserv crash
cfang
parents:
3906
diff
changeset
|
2176 |
assert(!valid(), "unsafe access"); |
d1617f46285d
6852078: HSX 14/16 in jdk 5.0: api/javax_management api/org_omg jck tests crashes or make tnameserv crash
cfang
parents:
3906
diff
changeset
|
2177 |
return; |
d1617f46285d
6852078: HSX 14/16 in jdk 5.0: api/javax_management api/org_omg jck tests crashes or make tnameserv crash
cfang
parents:
3906
diff
changeset
|
2178 |
} |
1 | 2179 |
for (int i = 0; i < 3; i++) { |
2180 |
if (!scaled_iv_plus_offset(adr->in(AddPNode::Offset))) { |
|
2181 |
assert(!valid(), "too complex"); |
|
2182 |
return; |
|
2183 |
} |
|
2184 |
adr = adr->in(AddPNode::Address); |
|
2185 |
if (base == adr || !adr->is_AddP()) { |
|
2186 |
break; // stop looking at addp's |
|
2187 |
} |
|
2188 |
} |
|
2189 |
_base = base; |
|
2190 |
_adr = adr; |
|
2191 |
assert(valid(), "Usable"); |
|
2192 |
} |
|
2193 |
||
2194 |
// Following is used to create a temporary object during |
|
2195 |
// the pattern match of an address expression. |
|
2196 |
SWPointer::SWPointer(SWPointer* p) : |
|
2197 |
_mem(p->_mem), _slp(p->_slp), _base(NULL), _adr(NULL), |
|
2198 |
_scale(0), _offset(0), _invar(NULL), _negate_invar(false) {} |
|
2199 |
||
2200 |
//------------------------scaled_iv_plus_offset-------------------- |
|
2201 |
// Match: k*iv + offset |
|
2202 |
// where: k is a constant that maybe zero, and |
|
2203 |
// offset is (k2 [+/- invariant]) where k2 maybe zero and invariant is optional |
|
2204 |
bool SWPointer::scaled_iv_plus_offset(Node* n) { |
|
2205 |
if (scaled_iv(n)) { |
|
2206 |
return true; |
|
2207 |
} |
|
2208 |
if (offset_plus_k(n)) { |
|
2209 |
return true; |
|
2210 |
} |
|
2211 |
int opc = n->Opcode(); |
|
2212 |
if (opc == Op_AddI) { |
|
2213 |
if (scaled_iv(n->in(1)) && offset_plus_k(n->in(2))) { |
|
2214 |
return true; |
|
2215 |
} |
|
2216 |
if (scaled_iv(n->in(2)) && offset_plus_k(n->in(1))) { |
|
2217 |
return true; |
|
2218 |
} |
|
2219 |
} else if (opc == Op_SubI) { |
|
2220 |
if (scaled_iv(n->in(1)) && offset_plus_k(n->in(2), true)) { |
|
2221 |
return true; |
|
2222 |
} |
|
2223 |
if (scaled_iv(n->in(2)) && offset_plus_k(n->in(1))) { |
|
2224 |
_scale *= -1; |
|
2225 |
return true; |
|
2226 |
} |
|
2227 |
} |
|
2228 |
return false; |
|
2229 |
} |
|
2230 |
||
2231 |
//----------------------------scaled_iv------------------------ |
|
2232 |
// Match: k*iv where k is a constant that's not zero |
|
2233 |
bool SWPointer::scaled_iv(Node* n) { |
|
2234 |
if (_scale != 0) { |
|
2235 |
return false; // already found a scale |
|
2236 |
} |
|
2237 |
if (n == iv()) { |
|
2238 |
_scale = 1; |
|
2239 |
return true; |
|
2240 |
} |
|
2241 |
int opc = n->Opcode(); |
|
2242 |
if (opc == Op_MulI) { |
|
2243 |
if (n->in(1) == iv() && n->in(2)->is_Con()) { |
|
2244 |
_scale = n->in(2)->get_int(); |
|
2245 |
return true; |
|
2246 |
} else if (n->in(2) == iv() && n->in(1)->is_Con()) { |
|
2247 |
_scale = n->in(1)->get_int(); |
|
2248 |
return true; |
|
2249 |
} |
|
2250 |
} else if (opc == Op_LShiftI) { |
|
2251 |
if (n->in(1) == iv() && n->in(2)->is_Con()) { |
|
2252 |
_scale = 1 << n->in(2)->get_int(); |
|
2253 |
return true; |
|
2254 |
} |
|
2255 |
} else if (opc == Op_ConvI2L) { |
|
2256 |
if (scaled_iv_plus_offset(n->in(1))) { |
|
2257 |
return true; |
|
2258 |
} |
|
2259 |
} else if (opc == Op_LShiftL) { |
|
2260 |
if (!has_iv() && _invar == NULL) { |
|
2261 |
// Need to preserve the current _offset value, so |
|
2262 |
// create a temporary object for this expression subtree. |
|
2263 |
// Hacky, so should re-engineer the address pattern match. |
|
2264 |
SWPointer tmp(this); |
|
2265 |
if (tmp.scaled_iv_plus_offset(n->in(1))) { |
|
2266 |
if (tmp._invar == NULL) { |
|
2267 |
int mult = 1 << n->in(2)->get_int(); |
|
2268 |
_scale = tmp._scale * mult; |
|
2269 |
_offset += tmp._offset * mult; |
|
2270 |
return true; |
|
2271 |
} |
|
2272 |
} |
|
2273 |
} |
|
2274 |
} |
|
2275 |
return false; |
|
2276 |
} |
|
2277 |
||
2278 |
//----------------------------offset_plus_k------------------------ |
|
2279 |
// Match: offset is (k [+/- invariant]) |
|
2280 |
// where k maybe zero and invariant is optional, but not both. |
|
2281 |
bool SWPointer::offset_plus_k(Node* n, bool negate) { |
|
2282 |
int opc = n->Opcode(); |
|
2283 |
if (opc == Op_ConI) { |
|
2284 |
_offset += negate ? -(n->get_int()) : n->get_int(); |
|
2285 |
return true; |
|
2286 |
} else if (opc == Op_ConL) { |
|
2287 |
// Okay if value fits into an int |
|
2288 |
const TypeLong* t = n->find_long_type(); |
|
2289 |
if (t->higher_equal(TypeLong::INT)) { |
|
2290 |
jlong loff = n->get_long(); |
|
2291 |
jint off = (jint)loff; |
|
2292 |
_offset += negate ? -off : loff; |
|
2293 |
return true; |
|
2294 |
} |
|
2295 |
return false; |
|
2296 |
} |
|
2297 |
if (_invar != NULL) return false; // already have an invariant |
|
2298 |
if (opc == Op_AddI) { |
|
2299 |
if (n->in(2)->is_Con() && invariant(n->in(1))) { |
|
2300 |
_negate_invar = negate; |
|
2301 |
_invar = n->in(1); |
|
2302 |
_offset += negate ? -(n->in(2)->get_int()) : n->in(2)->get_int(); |
|
2303 |
return true; |
|
2304 |
} else if (n->in(1)->is_Con() && invariant(n->in(2))) { |
|
2305 |
_offset += negate ? -(n->in(1)->get_int()) : n->in(1)->get_int(); |
|
2306 |
_negate_invar = negate; |
|
2307 |
_invar = n->in(2); |
|
2308 |
return true; |
|
2309 |
} |
|
2310 |
} |
|
2311 |
if (opc == Op_SubI) { |
|
2312 |
if (n->in(2)->is_Con() && invariant(n->in(1))) { |
|
2313 |
_negate_invar = negate; |
|
2314 |
_invar = n->in(1); |
|
2315 |
_offset += !negate ? -(n->in(2)->get_int()) : n->in(2)->get_int(); |
|
2316 |
return true; |
|
2317 |
} else if (n->in(1)->is_Con() && invariant(n->in(2))) { |
|
2318 |
_offset += negate ? -(n->in(1)->get_int()) : n->in(1)->get_int(); |
|
2319 |
_negate_invar = !negate; |
|
2320 |
_invar = n->in(2); |
|
2321 |
return true; |
|
2322 |
} |
|
2323 |
} |
|
2324 |
if (invariant(n)) { |
|
2325 |
_negate_invar = negate; |
|
2326 |
_invar = n; |
|
2327 |
return true; |
|
2328 |
} |
|
2329 |
return false; |
|
2330 |
} |
|
2331 |
||
2332 |
//----------------------------print------------------------ |
|
2333 |
void SWPointer::print() { |
|
2334 |
#ifndef PRODUCT |
|
2335 |
tty->print("base: %d adr: %d scale: %d offset: %d invar: %c%d\n", |
|
2336 |
_base != NULL ? _base->_idx : 0, |
|
2337 |
_adr != NULL ? _adr->_idx : 0, |
|
2338 |
_scale, _offset, |
|
2339 |
_negate_invar?'-':'+', |
|
2340 |
_invar != NULL ? _invar->_idx : 0); |
|
2341 |
#endif |
|
2342 |
} |
|
2343 |
||
2344 |
// ========================= OrderedPair ===================== |
|
2345 |
||
2346 |
const OrderedPair OrderedPair::initial; |
|
2347 |
||
2348 |
// ========================= SWNodeInfo ===================== |
|
2349 |
||
2350 |
const SWNodeInfo SWNodeInfo::initial; |
|
2351 |
||
2352 |
||
2353 |
// ============================ DepGraph =========================== |
|
2354 |
||
2355 |
//------------------------------make_node--------------------------- |
|
2356 |
// Make a new dependence graph node for an ideal node. |
|
2357 |
DepMem* DepGraph::make_node(Node* node) { |
|
2358 |
DepMem* m = new (_arena) DepMem(node); |
|
2359 |
if (node != NULL) { |
|
2360 |
assert(_map.at_grow(node->_idx) == NULL, "one init only"); |
|
2361 |
_map.at_put_grow(node->_idx, m); |
|
2362 |
} |
|
2363 |
return m; |
|
2364 |
} |
|
2365 |
||
2366 |
//------------------------------make_edge--------------------------- |
|
2367 |
// Make a new dependence graph edge from dpred -> dsucc |
|
2368 |
DepEdge* DepGraph::make_edge(DepMem* dpred, DepMem* dsucc) { |
|
2369 |
DepEdge* e = new (_arena) DepEdge(dpred, dsucc, dsucc->in_head(), dpred->out_head()); |
|
2370 |
dpred->set_out_head(e); |
|
2371 |
dsucc->set_in_head(e); |
|
2372 |
return e; |
|
2373 |
} |
|
2374 |
||
2375 |
// ========================== DepMem ======================== |
|
2376 |
||
2377 |
//------------------------------in_cnt--------------------------- |
|
2378 |
int DepMem::in_cnt() { |
|
2379 |
int ct = 0; |
|
2380 |
for (DepEdge* e = _in_head; e != NULL; e = e->next_in()) ct++; |
|
2381 |
return ct; |
|
2382 |
} |
|
2383 |
||
2384 |
//------------------------------out_cnt--------------------------- |
|
2385 |
int DepMem::out_cnt() { |
|
2386 |
int ct = 0; |
|
2387 |
for (DepEdge* e = _out_head; e != NULL; e = e->next_out()) ct++; |
|
2388 |
return ct; |
|
2389 |
} |
|
2390 |
||
2391 |
//------------------------------print----------------------------- |
|
2392 |
void DepMem::print() { |
|
2393 |
#ifndef PRODUCT |
|
2394 |
tty->print(" DepNode %d (", _node->_idx); |
|
2395 |
for (DepEdge* p = _in_head; p != NULL; p = p->next_in()) { |
|
2396 |
Node* pred = p->pred()->node(); |
|
2397 |
tty->print(" %d", pred != NULL ? pred->_idx : 0); |
|
2398 |
} |
|
2399 |
tty->print(") ["); |
|
2400 |
for (DepEdge* s = _out_head; s != NULL; s = s->next_out()) { |
|
2401 |
Node* succ = s->succ()->node(); |
|
2402 |
tty->print(" %d", succ != NULL ? succ->_idx : 0); |
|
2403 |
} |
|
2404 |
tty->print_cr(" ]"); |
|
2405 |
#endif |
|
2406 |
} |
|
2407 |
||
2408 |
// =========================== DepEdge ========================= |
|
2409 |
||
2410 |
//------------------------------DepPreds--------------------------- |
|
2411 |
void DepEdge::print() { |
|
2412 |
#ifndef PRODUCT |
|
2413 |
tty->print_cr("DepEdge: %d [ %d ]", _pred->node()->_idx, _succ->node()->_idx); |
|
2414 |
#endif |
|
2415 |
} |
|
2416 |
||
2417 |
// =========================== DepPreds ========================= |
|
2418 |
// Iterator over predecessor edges in the dependence graph. |
|
2419 |
||
2420 |
//------------------------------DepPreds--------------------------- |
|
2421 |
DepPreds::DepPreds(Node* n, DepGraph& dg) { |
|
2422 |
_n = n; |
|
2423 |
_done = false; |
|
2424 |
if (_n->is_Store() || _n->is_Load()) { |
|
2425 |
_next_idx = MemNode::Address; |
|
2426 |
_end_idx = n->req(); |
|
2427 |
_dep_next = dg.dep(_n)->in_head(); |
|
2428 |
} else if (_n->is_Mem()) { |
|
2429 |
_next_idx = 0; |
|
2430 |
_end_idx = 0; |
|
2431 |
_dep_next = dg.dep(_n)->in_head(); |
|
2432 |
} else { |
|
2433 |
_next_idx = 1; |
|
2434 |
_end_idx = _n->req(); |
|
2435 |
_dep_next = NULL; |
|
2436 |
} |
|
2437 |
next(); |
|
2438 |
} |
|
2439 |
||
2440 |
//------------------------------next--------------------------- |
|
2441 |
void DepPreds::next() { |
|
2442 |
if (_dep_next != NULL) { |
|
2443 |
_current = _dep_next->pred()->node(); |
|
2444 |
_dep_next = _dep_next->next_in(); |
|
2445 |
} else if (_next_idx < _end_idx) { |
|
2446 |
_current = _n->in(_next_idx++); |
|
2447 |
} else { |
|
2448 |
_done = true; |
|
2449 |
} |
|
2450 |
} |
|
2451 |
||
2452 |
// =========================== DepSuccs ========================= |
|
2453 |
// Iterator over successor edges in the dependence graph. |
|
2454 |
||
2455 |
//------------------------------DepSuccs--------------------------- |
|
2456 |
DepSuccs::DepSuccs(Node* n, DepGraph& dg) { |
|
2457 |
_n = n; |
|
2458 |
_done = false; |
|
2459 |
if (_n->is_Load()) { |
|
2460 |
_next_idx = 0; |
|
2461 |
_end_idx = _n->outcnt(); |
|
2462 |
_dep_next = dg.dep(_n)->out_head(); |
|
2463 |
} else if (_n->is_Mem() || _n->is_Phi() && _n->bottom_type() == Type::MEMORY) { |
|
2464 |
_next_idx = 0; |
|
2465 |
_end_idx = 0; |
|
2466 |
_dep_next = dg.dep(_n)->out_head(); |
|
2467 |
} else { |
|
2468 |
_next_idx = 0; |
|
2469 |
_end_idx = _n->outcnt(); |
|
2470 |
_dep_next = NULL; |
|
2471 |
} |
|
2472 |
next(); |
|
2473 |
} |
|
2474 |
||
2475 |
//-------------------------------next--------------------------- |
|
2476 |
void DepSuccs::next() { |
|
2477 |
if (_dep_next != NULL) { |
|
2478 |
_current = _dep_next->succ()->node(); |
|
2479 |
_dep_next = _dep_next->next_out(); |
|
2480 |
} else if (_next_idx < _end_idx) { |
|
2481 |
_current = _n->raw_out(_next_idx++); |
|
2482 |
} else { |
|
2483 |
_done = true; |
|
2484 |
} |
|
2485 |
} |