author | iveresov |
Fri, 02 Feb 2018 17:28:17 -0800 | |
changeset 48861 | 47f19ff9903c |
parent 48505 | 5db30620a3db |
child 49451 | e06f9607f370 |
permissions | -rw-r--r-- |
43972 | 1 |
/* |
2 |
* Copyright (c) 2012, 2016, Oracle and/or its affiliates. All rights reserved. |
|
3 |
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
|
4 |
* |
|
5 |
* This code is free software; you can redistribute it and/or modify it |
|
6 |
* under the terms of the GNU General Public License version 2 only, as |
|
7 |
* published by the Free Software Foundation. |
|
8 |
* |
|
9 |
* This code is distributed in the hope that it will be useful, but WITHOUT |
|
10 |
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
|
11 |
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
|
12 |
* version 2 for more details (a copy is included in the LICENSE file that |
|
13 |
* accompanied this code). |
|
14 |
* |
|
15 |
* You should have received a copy of the GNU General Public License version |
|
16 |
* 2 along with this work; if not, write to the Free Software Foundation, |
|
17 |
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
|
18 |
* |
|
19 |
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
|
20 |
* or visit www.oracle.com if you need additional information or have any |
|
21 |
* questions. |
|
22 |
*/ |
|
23 |
package org.graalvm.compiler.hotspot.replacements; |
|
24 |
||
46344 | 25 |
import static jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider.getArrayBaseOffset; |
43972 | 26 |
import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC; |
27 |
import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan; |
|
28 |
import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG; |
|
29 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION; |
|
30 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION; |
|
31 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION; |
|
32 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION; |
|
33 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION; |
|
34 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION; |
|
48505
5db30620a3db
8191362: [Graal] gc/g1/TestShrinkAuxiliaryData tests crash with "assert(check_klass_alignment(result)) failed: address not aligned"
thartmann
parents:
47216
diff
changeset
|
35 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayAllocationSize; |
43972 | 36 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset; |
37 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset; |
|
38 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.config; |
|
39 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader; |
|
40 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize; |
|
41 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized; |
|
42 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask; |
|
43 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift; |
|
44 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask; |
|
45 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift; |
|
46 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject; |
|
47 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset; |
|
48 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper; |
|
49 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd; |
|
50 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop; |
|
51 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord; |
|
52 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking; |
|
53 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB; |
|
54 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop; |
|
55 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop; |
|
46344 | 56 |
import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocations; |
57 |
import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocationsContext; |
|
46371 | 58 |
import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCastToSnippetReplaceeStamp; |
46344 | 59 |
import static org.graalvm.compiler.nodes.PiNode.piCastToSnippetReplaceeStamp; |
43972 | 60 |
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY; |
61 |
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY; |
|
62 |
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY; |
|
63 |
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; |
|
64 |
import static org.graalvm.compiler.replacements.ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED; |
|
48861 | 65 |
import static org.graalvm.compiler.replacements.ReplacementsUtil.runtimeAssert; |
43972 | 66 |
import static org.graalvm.compiler.replacements.ReplacementsUtil.staticAssert; |
67 |
import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER; |
|
68 |
import static org.graalvm.compiler.replacements.nodes.CStringConstant.cstring; |
|
69 |
import static org.graalvm.compiler.replacements.nodes.ExplodeLoopNode.explodeLoop; |
|
70 |
||
71 |
import org.graalvm.compiler.api.replacements.Fold; |
|
72 |
import org.graalvm.compiler.api.replacements.Snippet; |
|
73 |
import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter; |
|
74 |
import org.graalvm.compiler.api.replacements.Snippet.VarargsParameter; |
|
75 |
import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; |
|
76 |
import org.graalvm.compiler.core.common.type.StampFactory; |
|
46640 | 77 |
import org.graalvm.compiler.debug.DebugHandlersFactory; |
43972 | 78 |
import org.graalvm.compiler.debug.GraalError; |
79 |
import org.graalvm.compiler.graph.Node.ConstantNodeParameter; |
|
80 |
import org.graalvm.compiler.graph.Node.NodeIntrinsic; |
|
81 |
import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; |
|
82 |
import org.graalvm.compiler.hotspot.HotSpotBackend; |
|
83 |
import org.graalvm.compiler.hotspot.meta.HotSpotProviders; |
|
84 |
import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider; |
|
85 |
import org.graalvm.compiler.hotspot.nodes.DimensionsNode; |
|
86 |
import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyFixedNode; |
|
87 |
import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyNode; |
|
88 |
import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp; |
|
89 |
import org.graalvm.compiler.hotspot.word.KlassPointer; |
|
90 |
import org.graalvm.compiler.nodes.ConstantNode; |
|
91 |
import org.graalvm.compiler.nodes.DeoptimizeNode; |
|
46344 | 92 |
import org.graalvm.compiler.nodes.PiNode; |
43972 | 93 |
import org.graalvm.compiler.nodes.PrefetchAllocateNode; |
46344 | 94 |
import org.graalvm.compiler.nodes.SnippetAnchorNode; |
43972 | 95 |
import org.graalvm.compiler.nodes.StructuredGraph; |
96 |
import org.graalvm.compiler.nodes.ValueNode; |
|
97 |
import org.graalvm.compiler.nodes.debug.DynamicCounterNode; |
|
98 |
import org.graalvm.compiler.nodes.debug.VerifyHeapNode; |
|
99 |
import org.graalvm.compiler.nodes.extended.ForeignCallNode; |
|
100 |
import org.graalvm.compiler.nodes.extended.MembarNode; |
|
101 |
import org.graalvm.compiler.nodes.java.DynamicNewArrayNode; |
|
102 |
import org.graalvm.compiler.nodes.java.DynamicNewInstanceNode; |
|
103 |
import org.graalvm.compiler.nodes.java.NewArrayNode; |
|
104 |
import org.graalvm.compiler.nodes.java.NewInstanceNode; |
|
105 |
import org.graalvm.compiler.nodes.java.NewMultiArrayNode; |
|
106 |
import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode; |
|
107 |
import org.graalvm.compiler.nodes.spi.LoweringTool; |
|
108 |
import org.graalvm.compiler.nodes.util.GraphUtil; |
|
46344 | 109 |
import org.graalvm.compiler.options.OptionValues; |
43972 | 110 |
import org.graalvm.compiler.replacements.ReplacementsUtil; |
111 |
import org.graalvm.compiler.replacements.SnippetCounter; |
|
46344 | 112 |
import org.graalvm.compiler.replacements.SnippetCounter.Group; |
43972 | 113 |
import org.graalvm.compiler.replacements.SnippetTemplate; |
114 |
import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates; |
|
115 |
import org.graalvm.compiler.replacements.SnippetTemplate.Arguments; |
|
116 |
import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo; |
|
117 |
import org.graalvm.compiler.replacements.Snippets; |
|
118 |
import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode; |
|
119 |
import org.graalvm.compiler.word.Word; |
|
46551 | 120 |
import org.graalvm.word.LocationIdentity; |
121 |
import org.graalvm.word.WordFactory; |
|
43972 | 122 |
|
123 |
import jdk.vm.ci.code.CodeUtil; |
|
124 |
import jdk.vm.ci.code.MemoryBarriers; |
|
125 |
import jdk.vm.ci.code.Register; |
|
126 |
import jdk.vm.ci.code.TargetDescription; |
|
127 |
import jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider; |
|
128 |
import jdk.vm.ci.hotspot.HotSpotResolvedObjectType; |
|
129 |
import jdk.vm.ci.meta.DeoptimizationAction; |
|
130 |
import jdk.vm.ci.meta.DeoptimizationReason; |
|
131 |
import jdk.vm.ci.meta.JavaKind; |
|
132 |
import jdk.vm.ci.meta.ResolvedJavaType; |
|
133 |
||
134 |
/** |
|
135 |
* Snippets used for implementing NEW, ANEWARRAY and NEWARRAY. |
|
136 |
*/ |
|
137 |
public class NewObjectSnippets implements Snippets { |
|
138 |
||
139 |
enum ProfileContext { |
|
140 |
AllocatingMethod, |
|
141 |
InstanceOrArray, |
|
142 |
AllocatedType, |
|
143 |
AllocatedTypesInMethod, |
|
144 |
Total |
|
145 |
} |
|
146 |
||
147 |
@Fold |
|
46344 | 148 |
static String createName(String path, String typeContext, OptionValues options) { |
149 |
switch (ProfileAllocationsContext.getValue(options)) { |
|
43972 | 150 |
case AllocatingMethod: |
151 |
return ""; |
|
152 |
case InstanceOrArray: |
|
153 |
return path; |
|
154 |
case AllocatedType: |
|
155 |
case AllocatedTypesInMethod: |
|
156 |
return typeContext; |
|
157 |
case Total: |
|
158 |
return "bytes"; |
|
159 |
default: |
|
160 |
throw GraalError.shouldNotReachHere(); |
|
161 |
} |
|
162 |
} |
|
163 |
||
164 |
@Fold |
|
46344 | 165 |
static boolean doProfile(OptionValues options) { |
166 |
return ProfileAllocations.getValue(options); |
|
43972 | 167 |
} |
168 |
||
169 |
@Fold |
|
46344 | 170 |
static boolean withContext(OptionValues options) { |
171 |
ProfileContext context = ProfileAllocationsContext.getValue(options); |
|
43972 | 172 |
return context == ProfileContext.AllocatingMethod || context == ProfileContext.AllocatedTypesInMethod; |
173 |
} |
|
174 |
||
46344 | 175 |
protected static void profileAllocation(String path, long size, String typeContext, OptionValues options) { |
176 |
if (doProfile(options)) { |
|
177 |
String name = createName(path, typeContext, options); |
|
43972 | 178 |
|
46344 | 179 |
boolean context = withContext(options); |
43972 | 180 |
DynamicCounterNode.counter(name, "number of bytes allocated", size, context); |
181 |
DynamicCounterNode.counter(name, "number of allocations", 1, context); |
|
182 |
} |
|
183 |
} |
|
184 |
||
185 |
public static void emitPrefetchAllocate(Word address, boolean isArray) { |
|
186 |
GraalHotSpotVMConfig config = config(INJECTED_VMCONFIG); |
|
187 |
if (config.allocatePrefetchStyle > 0) { |
|
188 |
// Insert a prefetch for each allocation only on the fast-path |
|
189 |
// Generate several prefetch instructions. |
|
190 |
int lines = isArray ? config.allocatePrefetchLines : config.allocateInstancePrefetchLines; |
|
191 |
int stepSize = config.allocatePrefetchStepSize; |
|
192 |
int distance = config.allocatePrefetchDistance; |
|
193 |
ExplodeLoopNode.explodeLoop(); |
|
194 |
for (int i = 0; i < lines; i++) { |
|
195 |
PrefetchAllocateNode.prefetch(OffsetAddressNode.address(address, distance)); |
|
196 |
distance += stepSize; |
|
197 |
} |
|
198 |
} |
|
199 |
} |
|
200 |
||
201 |
@Snippet |
|
46344 | 202 |
public static Object allocateInstance(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents, |
203 |
@ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter OptionValues options, |
|
204 |
@ConstantParameter Counters counters) { |
|
205 |
return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, options, counters)); |
|
206 |
} |
|
207 |
||
208 |
public static Object allocateInstanceHelper(int size, KlassPointer hub, Word prototypeMarkWord, boolean fillContents, |
|
209 |
Register threadRegister, boolean constantSize, String typeContext, OptionValues options, Counters counters) { |
|
43972 | 210 |
Object result; |
211 |
Word thread = registerAsWord(threadRegister); |
|
212 |
Word top = readTlabTop(thread); |
|
213 |
Word end = readTlabEnd(thread); |
|
214 |
Word newTop = top.add(size); |
|
215 |
if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) { |
|
216 |
writeTlabTop(thread, newTop); |
|
217 |
emitPrefetchAllocate(newTop, false); |
|
46344 | 218 |
result = formatObject(hub, size, top, prototypeMarkWord, fillContents, constantSize, counters); |
43972 | 219 |
} else { |
46344 | 220 |
if (counters != null && counters.stub != null) { |
221 |
counters.stub.inc(); |
|
222 |
} |
|
43972 | 223 |
result = newInstance(HotSpotBackend.NEW_INSTANCE, hub); |
224 |
} |
|
46344 | 225 |
profileAllocation("instance", size, typeContext, options); |
226 |
return verifyOop(result); |
|
43972 | 227 |
} |
228 |
||
46371 | 229 |
@NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true) |
43972 | 230 |
public static native Object newInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub); |
231 |
||
232 |
@Snippet |
|
233 |
public static Object allocateInstancePIC(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents, |
|
46344 | 234 |
@ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter OptionValues options, |
235 |
@ConstantParameter Counters counters) { |
|
43972 | 236 |
// Klass must be initialized by the time the first instance is allocated, therefore we can |
237 |
// just load it from the corresponding cell and avoid the resolution check. We have to use a |
|
238 |
// fixed load though, to prevent it from floating above the initialization. |
|
239 |
KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub); |
|
46344 | 240 |
return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, picHub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, options, counters)); |
43972 | 241 |
} |
242 |
||
243 |
@Snippet |
|
46344 | 244 |
public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, |
245 |
@ConstantParameter OptionValues options, @ConstantParameter Counters counters) { |
|
246 |
if (probability(SLOW_PATH_PROBABILITY, type == null)) { |
|
247 |
DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); |
|
248 |
} |
|
249 |
Class<?> nonNullType = PiNode.piCastNonNullClass(type, SnippetAnchorNode.anchor()); |
|
250 |
||
251 |
if (probability(SLOW_PATH_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) { |
|
43972 | 252 |
DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); |
253 |
} |
|
254 |
||
46344 | 255 |
return PiNode.piCastToSnippetReplaceeStamp(allocateInstanceDynamicHelper(type, fillContents, threadRegister, options, counters, nonNullType)); |
256 |
} |
|
257 |
||
258 |
private static Object allocateInstanceDynamicHelper(Class<?> type, boolean fillContents, Register threadRegister, OptionValues options, Counters counters, Class<?> nonNullType) { |
|
259 |
KlassPointer hub = ClassGetHubNode.readClass(nonNullType); |
|
43972 | 260 |
if (probability(FAST_PATH_PROBABILITY, !hub.isNull())) { |
46344 | 261 |
KlassPointer nonNullHub = ClassGetHubNode.piCastNonNull(hub, SnippetAnchorNode.anchor()); |
262 |
||
263 |
if (probability(FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(nonNullHub))) { |
|
264 |
int layoutHelper = readLayoutHelper(nonNullHub); |
|
43972 | 265 |
/* |
266 |
* src/share/vm/oops/klass.hpp: For instances, layout helper is a positive number, |
|
267 |
* the instance size. This size is already passed through align_object_size and |
|
268 |
* scaled to bytes. The low order bit is set if instances of this class cannot be |
|
269 |
* allocated using the fastpath. |
|
270 |
*/ |
|
271 |
if (probability(FAST_PATH_PROBABILITY, (layoutHelper & 1) == 0)) { |
|
46344 | 272 |
Word prototypeMarkWord = nonNullHub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION); |
43972 | 273 |
/* |
274 |
* FIXME(je,ds): we should actually pass typeContext instead of "" but late |
|
275 |
* binding of parameters is not yet supported by the GraphBuilderPlugin system. |
|
276 |
*/ |
|
46344 | 277 |
return allocateInstanceHelper(layoutHelper, nonNullHub, prototypeMarkWord, fillContents, threadRegister, false, "", options, counters); |
43972 | 278 |
} |
279 |
} |
|
280 |
} |
|
281 |
return dynamicNewInstanceStub(type); |
|
282 |
} |
|
283 |
||
284 |
/** |
|
285 |
* Maximum array length for which fast path allocation is used. |
|
286 |
*/ |
|
287 |
public static final int MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH = 0x00FFFFFF; |
|
288 |
||
289 |
@Snippet |
|
290 |
public static Object allocatePrimitiveArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize, |
|
46344 | 291 |
@ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext, |
292 |
@ConstantParameter OptionValues options, @ConstantParameter Counters counters) { |
|
46551 | 293 |
// Primitive array types are eagerly pre-resolved. We can use a floating load. |
43972 | 294 |
KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub); |
46344 | 295 |
return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters); |
43972 | 296 |
} |
297 |
||
298 |
@Snippet |
|
299 |
public static Object allocateArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize, |
|
46344 | 300 |
@ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext, |
301 |
@ConstantParameter OptionValues options, @ConstantParameter Counters counters) { |
|
46566 | 302 |
// Array type would be resolved by dominating resolution. |
303 |
KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub); |
|
46344 | 304 |
return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters); |
43972 | 305 |
} |
306 |
||
307 |
@Snippet |
|
308 |
public static Object allocateArray(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize, |
|
46344 | 309 |
@ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext, |
310 |
@ConstantParameter OptionValues options, @ConstantParameter Counters counters) { |
|
311 |
Object result = allocateArrayImpl(hub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters); |
|
46371 | 312 |
return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length); |
43972 | 313 |
} |
314 |
||
46344 | 315 |
private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister, |
316 |
boolean maybeUnroll, String typeContext, boolean skipNegativeCheck, OptionValues options, Counters counters) { |
|
43972 | 317 |
Object result; |
48505
5db30620a3db
8191362: [Graal] gc/g1/TestShrinkAuxiliaryData tests crash with "assert(check_klass_alignment(result)) failed: address not aligned"
thartmann
parents:
47216
diff
changeset
|
318 |
int allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize); |
43972 | 319 |
Word thread = registerAsWord(threadRegister); |
320 |
Word top = readTlabTop(thread); |
|
321 |
Word end = readTlabEnd(thread); |
|
322 |
Word newTop = top.add(allocationSize); |
|
323 |
if (probability(FREQUENT_PROBABILITY, skipNegativeCheck || belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) && |
|
324 |
probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) { |
|
325 |
writeTlabTop(thread, newTop); |
|
326 |
emitPrefetchAllocate(newTop, true); |
|
46344 | 327 |
if (counters != null && counters.arrayLoopInit != null) { |
328 |
counters.arrayLoopInit.inc(); |
|
329 |
} |
|
330 |
result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, maybeUnroll, counters); |
|
43972 | 331 |
} else { |
332 |
result = newArray(HotSpotBackend.NEW_ARRAY, hub, length, fillContents); |
|
333 |
} |
|
46344 | 334 |
profileAllocation("array", allocationSize, typeContext, options); |
43972 | 335 |
return result; |
336 |
} |
|
337 |
||
46371 | 338 |
@NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true) |
43972 | 339 |
public static native Object newArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length, boolean fillContents); |
340 |
||
341 |
public static final ForeignCallDescriptor DYNAMIC_NEW_ARRAY = new ForeignCallDescriptor("dynamic_new_array", Object.class, Class.class, int.class); |
|
342 |
public static final ForeignCallDescriptor DYNAMIC_NEW_INSTANCE = new ForeignCallDescriptor("dynamic_new_instance", Object.class, Class.class); |
|
343 |
||
46371 | 344 |
@NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true) |
43972 | 345 |
public static native Object dynamicNewArrayStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType, int length); |
346 |
||
347 |
public static Object dynamicNewInstanceStub(Class<?> elementType) { |
|
348 |
return dynamicNewInstanceStubCall(DYNAMIC_NEW_INSTANCE, elementType); |
|
349 |
} |
|
350 |
||
46371 | 351 |
@NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true) |
43972 | 352 |
public static native Object dynamicNewInstanceStubCall(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType); |
353 |
||
354 |
@Snippet |
|
355 |
public static Object allocateArrayDynamic(Class<?> elementType, Class<?> voidClass, int length, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, |
|
46344 | 356 |
@ConstantParameter JavaKind knownElementKind, @ConstantParameter int knownLayoutHelper, Word prototypeMarkWord, @ConstantParameter OptionValues options, |
357 |
@ConstantParameter Counters counters) { |
|
358 |
Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, threadRegister, knownElementKind, knownLayoutHelper, prototypeMarkWord, options, counters); |
|
43972 | 359 |
return result; |
360 |
} |
|
361 |
||
362 |
private static Object allocateArrayDynamicImpl(Class<?> elementType, Class<?> voidClass, int length, boolean fillContents, Register threadRegister, JavaKind knownElementKind, |
|
46344 | 363 |
int knownLayoutHelper, Word prototypeMarkWord, OptionValues options, Counters counters) { |
43972 | 364 |
/* |
365 |
* We only need the dynamic check for void when we have no static information from |
|
366 |
* knownElementKind. |
|
367 |
*/ |
|
368 |
staticAssert(knownElementKind != JavaKind.Void, "unsupported knownElementKind"); |
|
369 |
if (knownElementKind == JavaKind.Illegal && probability(SLOW_PATH_PROBABILITY, elementType == null || DynamicNewArrayNode.throwsIllegalArgumentException(elementType, voidClass))) { |
|
370 |
DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); |
|
371 |
} |
|
372 |
||
373 |
KlassPointer klass = loadKlassFromObject(elementType, arrayKlassOffset(INJECTED_VMCONFIG), CLASS_ARRAY_KLASS_LOCATION); |
|
46344 | 374 |
if (klass.isNull()) { |
43972 | 375 |
DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); |
376 |
} |
|
46344 | 377 |
KlassPointer nonNullKlass = ClassGetHubNode.piCastNonNull(klass, SnippetAnchorNode.anchor()); |
378 |
||
379 |
if (length < 0) { |
|
380 |
DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); |
|
381 |
} |
|
48861 | 382 |
int layoutHelper; |
383 |
if (knownElementKind == JavaKind.Illegal) { |
|
384 |
layoutHelper = readLayoutHelper(nonNullKlass); |
|
385 |
} else { |
|
386 |
runtimeAssert(knownLayoutHelper == readLayoutHelper(nonNullKlass), "layout mismatch"); |
|
387 |
layoutHelper = knownLayoutHelper; |
|
388 |
} |
|
43972 | 389 |
//@formatter:off |
390 |
// from src/share/vm/oops/klass.hpp: |
|
391 |
// |
|
392 |
// For arrays, layout helper is a negative number, containing four |
|
393 |
// distinct bytes, as follows: |
|
394 |
// MSB:[tag, hsz, ebt, log2(esz)]:LSB |
|
395 |
// where: |
|
396 |
// tag is 0x80 if the elements are oops, 0xC0 if non-oops |
|
397 |
// hsz is array header size in bytes (i.e., offset of first element) |
|
398 |
// ebt is the BasicType of the elements |
|
399 |
// esz is the element size in bytes |
|
400 |
//@formatter:on |
|
401 |
||
402 |
int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG); |
|
403 |
int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG); |
|
404 |
||
46344 | 405 |
Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, false, "dynamic type", true, options, counters); |
46371 | 406 |
return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length); |
43972 | 407 |
} |
408 |
||
409 |
/** |
|
410 |
* Calls the runtime stub for implementing MULTIANEWARRAY. |
|
411 |
*/ |
|
412 |
@Snippet |
|
413 |
public static Object newmultiarray(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) { |
|
414 |
Word dims = DimensionsNode.allocaDimsArray(rank); |
|
415 |
ExplodeLoopNode.explodeLoop(); |
|
416 |
for (int i = 0; i < rank; i++) { |
|
46344 | 417 |
dims.writeInt(i * 4, dimensions[i], LocationIdentity.init()); |
43972 | 418 |
} |
419 |
return newArrayCall(HotSpotBackend.NEW_MULTI_ARRAY, hub, rank, dims); |
|
420 |
} |
|
421 |
||
422 |
@Snippet |
|
423 |
public static Object newmultiarrayPIC(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) { |
|
46566 | 424 |
// Array type would be resolved by dominating resolution. |
46509 | 425 |
KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub); |
426 |
return newmultiarray(picHub, rank, dimensions); |
|
43972 | 427 |
} |
428 |
||
46371 | 429 |
@NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true) |
43972 | 430 |
public static native Object newArrayCall(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims); |
431 |
||
432 |
/** |
|
433 |
* Maximum number of long stores to emit when zeroing an object with a constant size. Larger |
|
434 |
* objects have their bodies initialized in a loop. |
|
435 |
*/ |
|
436 |
private static final int MAX_UNROLLED_OBJECT_ZEROING_STORES = 8; |
|
437 |
||
438 |
/** |
|
439 |
* Zero uninitialized memory in a newly allocated object, unrolling as necessary and ensuring |
|
440 |
* that stores are aligned. |
|
441 |
* |
|
442 |
* @param size number of bytes to zero |
|
443 |
* @param memory beginning of object which is being zeroed |
|
444 |
* @param constantSize is {@code size} known to be constant in the snippet |
|
445 |
* @param startOffset offset to begin zeroing. May not be word aligned. |
|
446 |
* @param manualUnroll maximally unroll zeroing |
|
447 |
*/ |
|
46344 | 448 |
private static void zeroMemory(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) { |
449 |
fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, counters); |
|
43972 | 450 |
} |
451 |
||
46344 | 452 |
private static void fillMemory(long value, int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) { |
43972 | 453 |
ReplacementsUtil.runtimeAssert((size & 0x7) == 0, "unaligned object size"); |
454 |
int offset = startOffset; |
|
455 |
if ((offset & 0x7) != 0) { |
|
46344 | 456 |
memory.writeInt(offset, (int) value, LocationIdentity.init()); |
43972 | 457 |
offset += 4; |
458 |
} |
|
459 |
ReplacementsUtil.runtimeAssert((offset & 0x7) == 0, "unaligned offset"); |
|
460 |
if (manualUnroll && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) { |
|
461 |
ReplacementsUtil.staticAssert(!constantSize, "size shouldn't be constant at instantiation time"); |
|
462 |
// This case handles arrays of constant length. Instead of having a snippet variant for |
|
463 |
// each length, generate a chain of stores of maximum length. Once it's inlined the |
|
464 |
// break statement will trim excess stores. |
|
46344 | 465 |
if (counters != null && counters.instanceSeqInit != null) { |
466 |
counters.instanceSeqInit.inc(); |
|
43972 | 467 |
} |
46344 | 468 |
|
43972 | 469 |
explodeLoop(); |
470 |
for (int i = 0; i < MAX_UNROLLED_OBJECT_ZEROING_STORES; i++, offset += 8) { |
|
471 |
if (offset == size) { |
|
472 |
break; |
|
473 |
} |
|
46344 | 474 |
memory.initializeLong(offset, value, LocationIdentity.init()); |
43972 | 475 |
} |
476 |
} else { |
|
477 |
// Use Word instead of int to avoid extension to long in generated code |
|
46459 | 478 |
Word off = WordFactory.signed(offset); |
43972 | 479 |
if (constantSize && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) { |
46344 | 480 |
if (counters != null && counters.instanceSeqInit != null) { |
481 |
counters.instanceSeqInit.inc(); |
|
43972 | 482 |
} |
483 |
explodeLoop(); |
|
484 |
} else { |
|
46344 | 485 |
if (counters != null && counters.instanceLoopInit != null) { |
486 |
counters.instanceLoopInit.inc(); |
|
43972 | 487 |
} |
488 |
} |
|
489 |
for (; off.rawValue() < size; off = off.add(8)) { |
|
46344 | 490 |
memory.initializeLong(off, value, LocationIdentity.init()); |
43972 | 491 |
} |
492 |
} |
|
493 |
} |
|
494 |
||
495 |
/** |
|
496 |
* Fill uninitialized memory with garbage value in a newly allocated object, unrolling as |
|
497 |
* necessary and ensuring that stores are aligned. |
|
498 |
* |
|
499 |
* @param size number of bytes to zero |
|
500 |
* @param memory beginning of object which is being zeroed |
|
501 |
* @param constantSize is {@code size} known to be constant in the snippet |
|
502 |
* @param startOffset offset to begin zeroing. May not be word aligned. |
|
503 |
* @param manualUnroll maximally unroll zeroing |
|
504 |
*/ |
|
46344 | 505 |
private static void fillWithGarbage(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) { |
506 |
fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, counters); |
|
43972 | 507 |
} |
508 |
||
509 |
/** |
|
510 |
* Formats some allocated memory with an object header and zeroes out the rest. Disables asserts |
|
511 |
* since they can't be compiled in stubs. |
|
512 |
*/ |
|
513 |
public static Object formatObjectForStub(KlassPointer hub, int size, Word memory, Word compileTimePrototypeMarkWord) { |
|
46344 | 514 |
return formatObject(hub, size, memory, compileTimePrototypeMarkWord, true, false, null); |
43972 | 515 |
} |
516 |
||
517 |
/** |
|
518 |
* Formats some allocated memory with an object header and zeroes out the rest. |
|
519 |
*/ |
|
46344 | 520 |
protected static Object formatObject(KlassPointer hub, int size, Word memory, Word compileTimePrototypeMarkWord, boolean fillContents, boolean constantSize, Counters counters) { |
43972 | 521 |
Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord; |
522 |
initializeObjectHeader(memory, prototypeMarkWord, hub); |
|
523 |
if (fillContents) { |
|
46344 | 524 |
zeroMemory(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters); |
43972 | 525 |
} else if (REPLACEMENTS_ASSERTIONS_ENABLED) { |
46344 | 526 |
fillWithGarbage(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters); |
43972 | 527 |
} |
46344 | 528 |
MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init()); |
529 |
return memory.toObjectNonNull(); |
|
43972 | 530 |
} |
531 |
||
532 |
@Snippet |
|
533 |
protected static void verifyHeap(@ConstantParameter Register threadRegister) { |
|
534 |
Word thread = registerAsWord(threadRegister); |
|
535 |
Word topValue = readTlabTop(thread); |
|
46459 | 536 |
if (!topValue.equal(WordFactory.zero())) { |
43972 | 537 |
Word topValueContents = topValue.readWord(0, MARK_WORD_LOCATION); |
46459 | 538 |
if (topValueContents.equal(WordFactory.zero())) { |
43972 | 539 |
AssertionSnippets.vmMessageC(AssertionSnippets.ASSERTION_VM_MESSAGE_C, true, cstring("overzeroing of TLAB detected"), 0L, 0L, 0L); |
540 |
} |
|
541 |
} |
|
542 |
} |
|
543 |
||
544 |
/** |
|
545 |
* Formats some allocated memory with an object header and zeroes out the rest. |
|
546 |
*/ |
|
547 |
public static Object formatArray(KlassPointer hub, int allocationSize, int length, int headerSize, Word memory, Word prototypeMarkWord, boolean fillContents, boolean maybeUnroll, |
|
46344 | 548 |
Counters counters) { |
549 |
memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, LocationIdentity.init()); |
|
43972 | 550 |
/* |
551 |
* store hub last as the concurrent garbage collectors assume length is valid if hub field |
|
552 |
* is not null |
|
553 |
*/ |
|
554 |
initializeObjectHeader(memory, prototypeMarkWord, hub); |
|
555 |
if (fillContents) { |
|
46344 | 556 |
zeroMemory(allocationSize, memory, false, headerSize, maybeUnroll, counters); |
43972 | 557 |
} else if (REPLACEMENTS_ASSERTIONS_ENABLED) { |
46344 | 558 |
fillWithGarbage(allocationSize, memory, false, headerSize, maybeUnroll, counters); |
43972 | 559 |
} |
46344 | 560 |
MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init()); |
561 |
return memory.toObjectNonNull(); |
|
562 |
} |
|
563 |
||
564 |
static class Counters { |
|
565 |
Counters(SnippetCounter.Group.Factory factory) { |
|
566 |
Group newInstance = factory.createSnippetCounterGroup("NewInstance"); |
|
567 |
Group newArray = factory.createSnippetCounterGroup("NewArray"); |
|
568 |
instanceSeqInit = new SnippetCounter(newInstance, "tlabSeqInit", "TLAB alloc with unrolled zeroing"); |
|
569 |
instanceLoopInit = new SnippetCounter(newInstance, "tlabLoopInit", "TLAB alloc with zeroing in a loop"); |
|
570 |
arrayLoopInit = new SnippetCounter(newArray, "tlabLoopInit", "TLAB alloc with zeroing in a loop"); |
|
571 |
stub = new SnippetCounter(newInstance, "stub", "alloc and zeroing via stub"); |
|
572 |
} |
|
573 |
||
574 |
final SnippetCounter instanceSeqInit; |
|
575 |
final SnippetCounter instanceLoopInit; |
|
576 |
final SnippetCounter arrayLoopInit; |
|
577 |
final SnippetCounter stub; |
|
43972 | 578 |
} |
579 |
||
580 |
public static class Templates extends AbstractTemplates { |
|
581 |
||
46344 | 582 |
private final SnippetInfo allocateInstance = snippet(NewObjectSnippets.class, "allocateInstance", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION); |
583 |
private final SnippetInfo allocateInstancePIC = snippet(NewObjectSnippets.class, "allocateInstancePIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, |
|
43972 | 584 |
TLAB_END_LOCATION); |
46344 | 585 |
private final SnippetInfo allocateArray = snippet(NewObjectSnippets.class, "allocateArray", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION); |
586 |
private final SnippetInfo allocateArrayPIC = snippet(NewObjectSnippets.class, "allocateArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION); |
|
587 |
private final SnippetInfo allocatePrimitiveArrayPIC = snippet(NewObjectSnippets.class, "allocatePrimitiveArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, |
|
43972 | 588 |
TLAB_END_LOCATION); |
46344 | 589 |
private final SnippetInfo allocateArrayDynamic = snippet(NewObjectSnippets.class, "allocateArrayDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, |
43972 | 590 |
TLAB_END_LOCATION); |
46344 | 591 |
private final SnippetInfo allocateInstanceDynamic = snippet(NewObjectSnippets.class, "allocateInstanceDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, |
43972 | 592 |
TLAB_END_LOCATION); |
46344 | 593 |
private final SnippetInfo newmultiarray = snippet(NewObjectSnippets.class, "newmultiarray", TLAB_TOP_LOCATION, TLAB_END_LOCATION); |
594 |
private final SnippetInfo newmultiarrayPIC = snippet(NewObjectSnippets.class, "newmultiarrayPIC", TLAB_TOP_LOCATION, TLAB_END_LOCATION); |
|
43972 | 595 |
private final SnippetInfo verifyHeap = snippet(NewObjectSnippets.class, "verifyHeap"); |
596 |
private final GraalHotSpotVMConfig config; |
|
46344 | 597 |
private final Counters counters; |
43972 | 598 |
|
46640 | 599 |
public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target, |
600 |
GraalHotSpotVMConfig config) { |
|
601 |
super(options, factories, providers, providers.getSnippetReflection(), target); |
|
43972 | 602 |
this.config = config; |
46344 | 603 |
counters = new Counters(factory); |
43972 | 604 |
} |
605 |
||
606 |
/** |
|
607 |
* Lowers a {@link NewInstanceNode}. |
|
608 |
*/ |
|
609 |
public void lower(NewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) { |
|
610 |
StructuredGraph graph = newInstanceNode.graph(); |
|
611 |
HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newInstanceNode.instanceClass(); |
|
612 |
assert !type.isArray(); |
|
613 |
ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph); |
|
614 |
int size = instanceSize(type); |
|
615 |
||
46344 | 616 |
OptionValues localOptions = graph.getOptions(); |
617 |
SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? allocateInstancePIC : allocateInstance; |
|
43972 | 618 |
Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage()); |
619 |
args.addConst("size", size); |
|
620 |
args.add("hub", hub); |
|
621 |
args.add("prototypeMarkWord", type.prototypeMarkWord()); |
|
622 |
args.addConst("fillContents", newInstanceNode.fillContents()); |
|
623 |
args.addConst("threadRegister", registers.getThreadRegister()); |
|
624 |
args.addConst("constantSize", true); |
|
46344 | 625 |
args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? type.toJavaName(false) : ""); |
626 |
args.addConst("options", localOptions); |
|
627 |
args.addConst("counters", counters); |
|
43972 | 628 |
|
46640 | 629 |
SnippetTemplate template = template(graph.getDebug(), args); |
630 |
graph.getDebug().log("Lowering allocateInstance in %s: node=%s, template=%s, arguments=%s", graph, newInstanceNode, template, args); |
|
43972 | 631 |
template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args); |
632 |
} |
|
633 |
||
634 |
/** |
|
635 |
* Lowers a {@link NewArrayNode}. |
|
636 |
*/ |
|
637 |
public void lower(NewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) { |
|
638 |
StructuredGraph graph = newArrayNode.graph(); |
|
639 |
ResolvedJavaType elementType = newArrayNode.elementType(); |
|
640 |
HotSpotResolvedObjectType arrayType = (HotSpotResolvedObjectType) elementType.getArrayClass(); |
|
641 |
JavaKind elementKind = elementType.getJavaKind(); |
|
642 |
ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayType.klass(), providers.getMetaAccess(), graph); |
|
643 |
final int headerSize = getArrayBaseOffset(elementKind); |
|
644 |
int log2ElementSize = CodeUtil.log2(HotSpotJVMCIRuntimeProvider.getArrayIndexScale(elementKind)); |
|
645 |
||
46344 | 646 |
OptionValues localOptions = graph.getOptions(); |
43972 | 647 |
SnippetInfo snippet; |
46344 | 648 |
if (GeneratePIC.getValue(localOptions)) { |
43972 | 649 |
if (elementType.isPrimitive()) { |
650 |
snippet = allocatePrimitiveArrayPIC; |
|
651 |
} else { |
|
652 |
snippet = allocateArrayPIC; |
|
653 |
} |
|
654 |
} else { |
|
655 |
snippet = allocateArray; |
|
656 |
} |
|
657 |
||
658 |
Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage()); |
|
659 |
args.add("hub", hub); |
|
660 |
ValueNode length = newArrayNode.length(); |
|
661 |
args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length)); |
|
662 |
assert arrayType.prototypeMarkWord() == lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord() : "all array types are assumed to have the same prototypeMarkWord"; |
|
663 |
args.add("prototypeMarkWord", arrayType.prototypeMarkWord()); |
|
664 |
args.addConst("headerSize", headerSize); |
|
665 |
args.addConst("log2ElementSize", log2ElementSize); |
|
666 |
args.addConst("fillContents", newArrayNode.fillContents()); |
|
667 |
args.addConst("threadRegister", registers.getThreadRegister()); |
|
668 |
args.addConst("maybeUnroll", length.isConstant()); |
|
46344 | 669 |
args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? arrayType.toJavaName(false) : ""); |
670 |
args.addConst("options", localOptions); |
|
671 |
args.addConst("counters", counters); |
|
46640 | 672 |
SnippetTemplate template = template(graph.getDebug(), args); |
673 |
graph.getDebug().log("Lowering allocateArray in %s: node=%s, template=%s, arguments=%s", graph, newArrayNode, template, args); |
|
43972 | 674 |
template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args); |
675 |
} |
|
676 |
||
677 |
public void lower(DynamicNewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) { |
|
678 |
Arguments args = new Arguments(allocateInstanceDynamic, newInstanceNode.graph().getGuardsStage(), tool.getLoweringStage()); |
|
46344 | 679 |
OptionValues localOptions = newInstanceNode.getOptions(); |
43972 | 680 |
args.add("type", newInstanceNode.getInstanceType()); |
681 |
ValueNode classClass = newInstanceNode.getClassClass(); |
|
682 |
assert classClass != null; |
|
683 |
args.add("classClass", classClass); |
|
684 |
args.addConst("fillContents", newInstanceNode.fillContents()); |
|
685 |
args.addConst("threadRegister", registers.getThreadRegister()); |
|
46344 | 686 |
args.addConst("options", localOptions); |
687 |
args.addConst("counters", counters); |
|
43972 | 688 |
|
46640 | 689 |
SnippetTemplate template = template(newInstanceNode.getDebug(), args); |
43972 | 690 |
template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args); |
691 |
} |
|
692 |
||
693 |
public void lower(DynamicNewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) { |
|
694 |
StructuredGraph graph = newArrayNode.graph(); |
|
46344 | 695 |
OptionValues localOptions = graph.getOptions(); |
43972 | 696 |
Arguments args = new Arguments(allocateArrayDynamic, newArrayNode.graph().getGuardsStage(), tool.getLoweringStage()); |
697 |
args.add("elementType", newArrayNode.getElementType()); |
|
698 |
ValueNode voidClass = newArrayNode.getVoidClass(); |
|
699 |
assert voidClass != null; |
|
700 |
args.add("voidClass", voidClass); |
|
701 |
ValueNode length = newArrayNode.length(); |
|
702 |
args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length)); |
|
703 |
args.addConst("fillContents", newArrayNode.fillContents()); |
|
704 |
args.addConst("threadRegister", registers.getThreadRegister()); |
|
705 |
/* |
|
706 |
* We use Kind.Illegal as a marker value instead of null because constant snippet |
|
707 |
* parameters cannot be null. |
|
708 |
*/ |
|
709 |
args.addConst("knownElementKind", newArrayNode.getKnownElementKind() == null ? JavaKind.Illegal : newArrayNode.getKnownElementKind()); |
|
710 |
if (newArrayNode.getKnownElementKind() != null) { |
|
711 |
args.addConst("knownLayoutHelper", lookupArrayClass(tool, newArrayNode.getKnownElementKind()).layoutHelper()); |
|
712 |
} else { |
|
713 |
args.addConst("knownLayoutHelper", 0); |
|
714 |
} |
|
715 |
args.add("prototypeMarkWord", lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord()); |
|
46344 | 716 |
args.addConst("options", localOptions); |
717 |
args.addConst("counters", counters); |
|
46640 | 718 |
SnippetTemplate template = template(graph.getDebug(), args); |
43972 | 719 |
template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args); |
720 |
} |
|
721 |
||
722 |
private static HotSpotResolvedObjectType lookupArrayClass(LoweringTool tool, JavaKind kind) { |
|
723 |
return (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(kind == JavaKind.Object ? Object.class : kind.toJavaClass()).getArrayClass(); |
|
724 |
} |
|
725 |
||
726 |
public void lower(NewMultiArrayNode newmultiarrayNode, LoweringTool tool) { |
|
727 |
StructuredGraph graph = newmultiarrayNode.graph(); |
|
46344 | 728 |
OptionValues localOptions = graph.getOptions(); |
43972 | 729 |
int rank = newmultiarrayNode.dimensionCount(); |
730 |
ValueNode[] dims = new ValueNode[rank]; |
|
731 |
for (int i = 0; i < newmultiarrayNode.dimensionCount(); i++) { |
|
732 |
dims[i] = newmultiarrayNode.dimension(i); |
|
733 |
} |
|
734 |
HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newmultiarrayNode.type(); |
|
735 |
ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph); |
|
736 |
||
46344 | 737 |
SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? newmultiarrayPIC : newmultiarray; |
43972 | 738 |
Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage()); |
739 |
args.add("hub", hub); |
|
740 |
args.addConst("rank", rank); |
|
741 |
args.addVarargs("dimensions", int.class, StampFactory.forKind(JavaKind.Int), dims); |
|
46640 | 742 |
template(newmultiarrayNode.getDebug(), args).instantiate(providers.getMetaAccess(), newmultiarrayNode, DEFAULT_REPLACER, args); |
43972 | 743 |
} |
744 |
||
745 |
private static int instanceSize(HotSpotResolvedObjectType type) { |
|
746 |
int size = type.instanceSize(); |
|
747 |
assert size >= 0; |
|
748 |
return size; |
|
749 |
} |
|
750 |
||
751 |
public void lower(VerifyHeapNode verifyHeapNode, HotSpotRegistersProvider registers, LoweringTool tool) { |
|
752 |
if (config.cAssertions) { |
|
753 |
Arguments args = new Arguments(verifyHeap, verifyHeapNode.graph().getGuardsStage(), tool.getLoweringStage()); |
|
754 |
args.addConst("threadRegister", registers.getThreadRegister()); |
|
755 |
||
46640 | 756 |
SnippetTemplate template = template(verifyHeapNode.getDebug(), args); |
43972 | 757 |
template.instantiate(providers.getMetaAccess(), verifyHeapNode, DEFAULT_REPLACER, args); |
758 |
} else { |
|
759 |
GraphUtil.removeFixedWithUnusedInputs(verifyHeapNode); |
|
760 |
} |
|
761 |
} |
|
762 |
} |
|
763 |
} |