43972
|
1 |
/*
|
|
2 |
* Copyright (c) 2012, 2016, Oracle and/or its affiliates. All rights reserved.
|
|
3 |
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
|
|
4 |
*
|
|
5 |
* This code is free software; you can redistribute it and/or modify it
|
|
6 |
* under the terms of the GNU General Public License version 2 only, as
|
|
7 |
* published by the Free Software Foundation.
|
|
8 |
*
|
|
9 |
* This code is distributed in the hope that it will be useful, but WITHOUT
|
|
10 |
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
11 |
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
12 |
* version 2 for more details (a copy is included in the LICENSE file that
|
|
13 |
* accompanied this code).
|
|
14 |
*
|
|
15 |
* You should have received a copy of the GNU General Public License version
|
|
16 |
* 2 along with this work; if not, write to the Free Software Foundation,
|
|
17 |
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
18 |
*
|
|
19 |
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
|
|
20 |
* or visit www.oracle.com if you need additional information or have any
|
|
21 |
* questions.
|
|
22 |
*/
|
|
23 |
package org.graalvm.compiler.hotspot.replacements;
|
|
24 |
|
46344
|
25 |
import static jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider.getArrayBaseOffset;
|
|
26 |
import static jdk.vm.ci.hotspot.HotSpotMetaAccessProvider.computeArrayAllocationSize;
|
43972
|
27 |
import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
|
|
28 |
import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan;
|
|
29 |
import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
|
|
30 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION;
|
|
31 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;
|
|
32 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
|
|
33 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
|
|
34 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION;
|
|
35 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION;
|
|
36 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset;
|
|
37 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset;
|
|
38 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.config;
|
|
39 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader;
|
|
40 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize;
|
|
41 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
|
|
42 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask;
|
|
43 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift;
|
|
44 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask;
|
|
45 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift;
|
|
46 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject;
|
|
47 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
|
|
48 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
|
|
49 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
|
|
50 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
|
|
51 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
|
|
52 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking;
|
|
53 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
|
|
54 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
|
|
55 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize;
|
|
56 |
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
|
46344
|
57 |
import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocations;
|
|
58 |
import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocationsContext;
|
46371
|
59 |
import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCastToSnippetReplaceeStamp;
|
46344
|
60 |
import static org.graalvm.compiler.nodes.PiNode.piCastToSnippetReplaceeStamp;
|
43972
|
61 |
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
|
|
62 |
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
|
|
63 |
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
|
|
64 |
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
|
|
65 |
import static org.graalvm.compiler.replacements.ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED;
|
|
66 |
import static org.graalvm.compiler.replacements.ReplacementsUtil.staticAssert;
|
|
67 |
import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
|
|
68 |
import static org.graalvm.compiler.replacements.nodes.CStringConstant.cstring;
|
|
69 |
import static org.graalvm.compiler.replacements.nodes.ExplodeLoopNode.explodeLoop;
|
|
70 |
|
|
71 |
import org.graalvm.compiler.api.replacements.Fold;
|
|
72 |
import org.graalvm.compiler.api.replacements.Snippet;
|
|
73 |
import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
|
|
74 |
import org.graalvm.compiler.api.replacements.Snippet.VarargsParameter;
|
|
75 |
import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
|
|
76 |
import org.graalvm.compiler.core.common.type.StampFactory;
|
46640
|
77 |
import org.graalvm.compiler.debug.DebugHandlersFactory;
|
43972
|
78 |
import org.graalvm.compiler.debug.GraalError;
|
|
79 |
import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
|
|
80 |
import org.graalvm.compiler.graph.Node.NodeIntrinsic;
|
|
81 |
import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
|
|
82 |
import org.graalvm.compiler.hotspot.HotSpotBackend;
|
|
83 |
import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
|
|
84 |
import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
|
|
85 |
import org.graalvm.compiler.hotspot.nodes.DimensionsNode;
|
|
86 |
import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyFixedNode;
|
|
87 |
import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyNode;
|
|
88 |
import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
|
|
89 |
import org.graalvm.compiler.hotspot.word.KlassPointer;
|
|
90 |
import org.graalvm.compiler.nodes.ConstantNode;
|
|
91 |
import org.graalvm.compiler.nodes.DeoptimizeNode;
|
46344
|
92 |
import org.graalvm.compiler.nodes.PiNode;
|
43972
|
93 |
import org.graalvm.compiler.nodes.PrefetchAllocateNode;
|
46344
|
94 |
import org.graalvm.compiler.nodes.SnippetAnchorNode;
|
43972
|
95 |
import org.graalvm.compiler.nodes.StructuredGraph;
|
|
96 |
import org.graalvm.compiler.nodes.ValueNode;
|
|
97 |
import org.graalvm.compiler.nodes.debug.DynamicCounterNode;
|
|
98 |
import org.graalvm.compiler.nodes.debug.VerifyHeapNode;
|
|
99 |
import org.graalvm.compiler.nodes.extended.ForeignCallNode;
|
|
100 |
import org.graalvm.compiler.nodes.extended.MembarNode;
|
|
101 |
import org.graalvm.compiler.nodes.java.DynamicNewArrayNode;
|
|
102 |
import org.graalvm.compiler.nodes.java.DynamicNewInstanceNode;
|
|
103 |
import org.graalvm.compiler.nodes.java.NewArrayNode;
|
|
104 |
import org.graalvm.compiler.nodes.java.NewInstanceNode;
|
|
105 |
import org.graalvm.compiler.nodes.java.NewMultiArrayNode;
|
|
106 |
import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
|
|
107 |
import org.graalvm.compiler.nodes.spi.LoweringTool;
|
|
108 |
import org.graalvm.compiler.nodes.util.GraphUtil;
|
46344
|
109 |
import org.graalvm.compiler.options.OptionValues;
|
43972
|
110 |
import org.graalvm.compiler.replacements.ReplacementsUtil;
|
|
111 |
import org.graalvm.compiler.replacements.SnippetCounter;
|
46344
|
112 |
import org.graalvm.compiler.replacements.SnippetCounter.Group;
|
43972
|
113 |
import org.graalvm.compiler.replacements.SnippetTemplate;
|
|
114 |
import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
|
|
115 |
import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
|
|
116 |
import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
|
|
117 |
import org.graalvm.compiler.replacements.Snippets;
|
|
118 |
import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode;
|
|
119 |
import org.graalvm.compiler.word.Word;
|
46551
|
120 |
import org.graalvm.word.LocationIdentity;
|
|
121 |
import org.graalvm.word.WordFactory;
|
43972
|
122 |
|
|
123 |
import jdk.vm.ci.code.CodeUtil;
|
|
124 |
import jdk.vm.ci.code.MemoryBarriers;
|
|
125 |
import jdk.vm.ci.code.Register;
|
|
126 |
import jdk.vm.ci.code.TargetDescription;
|
|
127 |
import jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider;
|
|
128 |
import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
|
|
129 |
import jdk.vm.ci.meta.DeoptimizationAction;
|
|
130 |
import jdk.vm.ci.meta.DeoptimizationReason;
|
|
131 |
import jdk.vm.ci.meta.JavaKind;
|
|
132 |
import jdk.vm.ci.meta.ResolvedJavaType;
|
|
133 |
|
|
134 |
/**
|
|
135 |
* Snippets used for implementing NEW, ANEWARRAY and NEWARRAY.
|
|
136 |
*/
|
|
137 |
public class NewObjectSnippets implements Snippets {
|
|
138 |
|
|
139 |
enum ProfileContext {
|
|
140 |
AllocatingMethod,
|
|
141 |
InstanceOrArray,
|
|
142 |
AllocatedType,
|
|
143 |
AllocatedTypesInMethod,
|
|
144 |
Total
|
|
145 |
}
|
|
146 |
|
|
147 |
@Fold
|
46344
|
148 |
static String createName(String path, String typeContext, OptionValues options) {
|
|
149 |
switch (ProfileAllocationsContext.getValue(options)) {
|
43972
|
150 |
case AllocatingMethod:
|
|
151 |
return "";
|
|
152 |
case InstanceOrArray:
|
|
153 |
return path;
|
|
154 |
case AllocatedType:
|
|
155 |
case AllocatedTypesInMethod:
|
|
156 |
return typeContext;
|
|
157 |
case Total:
|
|
158 |
return "bytes";
|
|
159 |
default:
|
|
160 |
throw GraalError.shouldNotReachHere();
|
|
161 |
}
|
|
162 |
}
|
|
163 |
|
|
164 |
@Fold
|
46344
|
165 |
static boolean doProfile(OptionValues options) {
|
|
166 |
return ProfileAllocations.getValue(options);
|
43972
|
167 |
}
|
|
168 |
|
|
169 |
@Fold
|
46344
|
170 |
static boolean withContext(OptionValues options) {
|
|
171 |
ProfileContext context = ProfileAllocationsContext.getValue(options);
|
43972
|
172 |
return context == ProfileContext.AllocatingMethod || context == ProfileContext.AllocatedTypesInMethod;
|
|
173 |
}
|
|
174 |
|
46344
|
175 |
protected static void profileAllocation(String path, long size, String typeContext, OptionValues options) {
|
|
176 |
if (doProfile(options)) {
|
|
177 |
String name = createName(path, typeContext, options);
|
43972
|
178 |
|
46344
|
179 |
boolean context = withContext(options);
|
43972
|
180 |
DynamicCounterNode.counter(name, "number of bytes allocated", size, context);
|
|
181 |
DynamicCounterNode.counter(name, "number of allocations", 1, context);
|
|
182 |
}
|
|
183 |
}
|
|
184 |
|
|
185 |
public static void emitPrefetchAllocate(Word address, boolean isArray) {
|
|
186 |
GraalHotSpotVMConfig config = config(INJECTED_VMCONFIG);
|
|
187 |
if (config.allocatePrefetchStyle > 0) {
|
|
188 |
// Insert a prefetch for each allocation only on the fast-path
|
|
189 |
// Generate several prefetch instructions.
|
|
190 |
int lines = isArray ? config.allocatePrefetchLines : config.allocateInstancePrefetchLines;
|
|
191 |
int stepSize = config.allocatePrefetchStepSize;
|
|
192 |
int distance = config.allocatePrefetchDistance;
|
|
193 |
ExplodeLoopNode.explodeLoop();
|
|
194 |
for (int i = 0; i < lines; i++) {
|
|
195 |
PrefetchAllocateNode.prefetch(OffsetAddressNode.address(address, distance));
|
|
196 |
distance += stepSize;
|
|
197 |
}
|
|
198 |
}
|
|
199 |
}
|
|
200 |
|
|
201 |
@Snippet
|
46344
|
202 |
public static Object allocateInstance(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
|
|
203 |
@ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter OptionValues options,
|
|
204 |
@ConstantParameter Counters counters) {
|
|
205 |
return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, options, counters));
|
|
206 |
}
|
|
207 |
|
|
208 |
public static Object allocateInstanceHelper(int size, KlassPointer hub, Word prototypeMarkWord, boolean fillContents,
|
|
209 |
Register threadRegister, boolean constantSize, String typeContext, OptionValues options, Counters counters) {
|
43972
|
210 |
Object result;
|
|
211 |
Word thread = registerAsWord(threadRegister);
|
|
212 |
Word top = readTlabTop(thread);
|
|
213 |
Word end = readTlabEnd(thread);
|
|
214 |
Word newTop = top.add(size);
|
|
215 |
if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
|
|
216 |
writeTlabTop(thread, newTop);
|
|
217 |
emitPrefetchAllocate(newTop, false);
|
46344
|
218 |
result = formatObject(hub, size, top, prototypeMarkWord, fillContents, constantSize, counters);
|
43972
|
219 |
} else {
|
46344
|
220 |
if (counters != null && counters.stub != null) {
|
|
221 |
counters.stub.inc();
|
|
222 |
}
|
43972
|
223 |
result = newInstance(HotSpotBackend.NEW_INSTANCE, hub);
|
|
224 |
}
|
46344
|
225 |
profileAllocation("instance", size, typeContext, options);
|
|
226 |
return verifyOop(result);
|
43972
|
227 |
}
|
|
228 |
|
46371
|
229 |
@NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
|
43972
|
230 |
public static native Object newInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
|
|
231 |
|
|
232 |
@Snippet
|
|
233 |
public static Object allocateInstancePIC(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
|
46344
|
234 |
@ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter OptionValues options,
|
|
235 |
@ConstantParameter Counters counters) {
|
43972
|
236 |
// Klass must be initialized by the time the first instance is allocated, therefore we can
|
|
237 |
// just load it from the corresponding cell and avoid the resolution check. We have to use a
|
|
238 |
// fixed load though, to prevent it from floating above the initialization.
|
|
239 |
KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
|
46344
|
240 |
return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, picHub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, options, counters));
|
43972
|
241 |
}
|
|
242 |
|
|
243 |
@Snippet
|
46344
|
244 |
public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
|
|
245 |
@ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
|
|
246 |
if (probability(SLOW_PATH_PROBABILITY, type == null)) {
|
|
247 |
DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
|
|
248 |
}
|
|
249 |
Class<?> nonNullType = PiNode.piCastNonNullClass(type, SnippetAnchorNode.anchor());
|
|
250 |
|
|
251 |
if (probability(SLOW_PATH_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) {
|
43972
|
252 |
DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
|
|
253 |
}
|
|
254 |
|
46344
|
255 |
return PiNode.piCastToSnippetReplaceeStamp(allocateInstanceDynamicHelper(type, fillContents, threadRegister, options, counters, nonNullType));
|
|
256 |
}
|
|
257 |
|
|
258 |
private static Object allocateInstanceDynamicHelper(Class<?> type, boolean fillContents, Register threadRegister, OptionValues options, Counters counters, Class<?> nonNullType) {
|
|
259 |
KlassPointer hub = ClassGetHubNode.readClass(nonNullType);
|
43972
|
260 |
if (probability(FAST_PATH_PROBABILITY, !hub.isNull())) {
|
46344
|
261 |
KlassPointer nonNullHub = ClassGetHubNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
|
|
262 |
|
|
263 |
if (probability(FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(nonNullHub))) {
|
|
264 |
int layoutHelper = readLayoutHelper(nonNullHub);
|
43972
|
265 |
/*
|
|
266 |
* src/share/vm/oops/klass.hpp: For instances, layout helper is a positive number,
|
|
267 |
* the instance size. This size is already passed through align_object_size and
|
|
268 |
* scaled to bytes. The low order bit is set if instances of this class cannot be
|
|
269 |
* allocated using the fastpath.
|
|
270 |
*/
|
|
271 |
if (probability(FAST_PATH_PROBABILITY, (layoutHelper & 1) == 0)) {
|
46344
|
272 |
Word prototypeMarkWord = nonNullHub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
|
43972
|
273 |
/*
|
|
274 |
* FIXME(je,ds): we should actually pass typeContext instead of "" but late
|
|
275 |
* binding of parameters is not yet supported by the GraphBuilderPlugin system.
|
|
276 |
*/
|
46344
|
277 |
return allocateInstanceHelper(layoutHelper, nonNullHub, prototypeMarkWord, fillContents, threadRegister, false, "", options, counters);
|
43972
|
278 |
}
|
|
279 |
}
|
|
280 |
}
|
|
281 |
return dynamicNewInstanceStub(type);
|
|
282 |
}
|
|
283 |
|
|
284 |
/**
|
|
285 |
* Maximum array length for which fast path allocation is used.
|
|
286 |
*/
|
|
287 |
public static final int MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH = 0x00FFFFFF;
|
|
288 |
|
|
289 |
@Snippet
|
|
290 |
public static Object allocatePrimitiveArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
|
46344
|
291 |
@ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
|
|
292 |
@ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
|
46551
|
293 |
// Primitive array types are eagerly pre-resolved. We can use a floating load.
|
43972
|
294 |
KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
|
46344
|
295 |
return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
|
43972
|
296 |
}
|
|
297 |
|
|
298 |
@Snippet
|
|
299 |
public static Object allocateArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
|
46344
|
300 |
@ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
|
|
301 |
@ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
|
46566
|
302 |
// Array type would be resolved by dominating resolution.
|
|
303 |
KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
|
46344
|
304 |
return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
|
43972
|
305 |
}
|
|
306 |
|
|
307 |
@Snippet
|
|
308 |
public static Object allocateArray(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
|
46344
|
309 |
@ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
|
|
310 |
@ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
|
|
311 |
Object result = allocateArrayImpl(hub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
|
46371
|
312 |
return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
|
43972
|
313 |
}
|
|
314 |
|
46344
|
315 |
private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister,
|
|
316 |
boolean maybeUnroll, String typeContext, boolean skipNegativeCheck, OptionValues options, Counters counters) {
|
43972
|
317 |
Object result;
|
|
318 |
int alignment = wordSize();
|
|
319 |
int allocationSize = computeArrayAllocationSize(length, alignment, headerSize, log2ElementSize);
|
|
320 |
Word thread = registerAsWord(threadRegister);
|
|
321 |
Word top = readTlabTop(thread);
|
|
322 |
Word end = readTlabEnd(thread);
|
|
323 |
Word newTop = top.add(allocationSize);
|
|
324 |
if (probability(FREQUENT_PROBABILITY, skipNegativeCheck || belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&
|
|
325 |
probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
|
|
326 |
writeTlabTop(thread, newTop);
|
|
327 |
emitPrefetchAllocate(newTop, true);
|
46344
|
328 |
if (counters != null && counters.arrayLoopInit != null) {
|
|
329 |
counters.arrayLoopInit.inc();
|
|
330 |
}
|
|
331 |
result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, maybeUnroll, counters);
|
43972
|
332 |
} else {
|
|
333 |
result = newArray(HotSpotBackend.NEW_ARRAY, hub, length, fillContents);
|
|
334 |
}
|
46344
|
335 |
profileAllocation("array", allocationSize, typeContext, options);
|
43972
|
336 |
return result;
|
|
337 |
}
|
|
338 |
|
46371
|
339 |
@NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
|
43972
|
340 |
public static native Object newArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length, boolean fillContents);
|
|
341 |
|
|
342 |
public static final ForeignCallDescriptor DYNAMIC_NEW_ARRAY = new ForeignCallDescriptor("dynamic_new_array", Object.class, Class.class, int.class);
|
|
343 |
public static final ForeignCallDescriptor DYNAMIC_NEW_INSTANCE = new ForeignCallDescriptor("dynamic_new_instance", Object.class, Class.class);
|
|
344 |
|
46371
|
345 |
@NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
|
43972
|
346 |
public static native Object dynamicNewArrayStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType, int length);
|
|
347 |
|
|
348 |
public static Object dynamicNewInstanceStub(Class<?> elementType) {
|
|
349 |
return dynamicNewInstanceStubCall(DYNAMIC_NEW_INSTANCE, elementType);
|
|
350 |
}
|
|
351 |
|
46371
|
352 |
@NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
|
43972
|
353 |
public static native Object dynamicNewInstanceStubCall(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
|
|
354 |
|
|
355 |
@Snippet
|
|
356 |
public static Object allocateArrayDynamic(Class<?> elementType, Class<?> voidClass, int length, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
|
46344
|
357 |
@ConstantParameter JavaKind knownElementKind, @ConstantParameter int knownLayoutHelper, Word prototypeMarkWord, @ConstantParameter OptionValues options,
|
|
358 |
@ConstantParameter Counters counters) {
|
|
359 |
Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, threadRegister, knownElementKind, knownLayoutHelper, prototypeMarkWord, options, counters);
|
43972
|
360 |
return result;
|
|
361 |
}
|
|
362 |
|
|
363 |
private static Object allocateArrayDynamicImpl(Class<?> elementType, Class<?> voidClass, int length, boolean fillContents, Register threadRegister, JavaKind knownElementKind,
|
46344
|
364 |
int knownLayoutHelper, Word prototypeMarkWord, OptionValues options, Counters counters) {
|
43972
|
365 |
/*
|
|
366 |
* We only need the dynamic check for void when we have no static information from
|
|
367 |
* knownElementKind.
|
|
368 |
*/
|
|
369 |
staticAssert(knownElementKind != JavaKind.Void, "unsupported knownElementKind");
|
|
370 |
if (knownElementKind == JavaKind.Illegal && probability(SLOW_PATH_PROBABILITY, elementType == null || DynamicNewArrayNode.throwsIllegalArgumentException(elementType, voidClass))) {
|
|
371 |
DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
|
|
372 |
}
|
|
373 |
|
|
374 |
KlassPointer klass = loadKlassFromObject(elementType, arrayKlassOffset(INJECTED_VMCONFIG), CLASS_ARRAY_KLASS_LOCATION);
|
46344
|
375 |
if (klass.isNull()) {
|
43972
|
376 |
DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
|
|
377 |
}
|
46344
|
378 |
KlassPointer nonNullKlass = ClassGetHubNode.piCastNonNull(klass, SnippetAnchorNode.anchor());
|
|
379 |
|
|
380 |
if (length < 0) {
|
|
381 |
DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
|
|
382 |
}
|
|
383 |
int layoutHelper = knownElementKind != JavaKind.Illegal ? knownLayoutHelper : readLayoutHelper(nonNullKlass);
|
43972
|
384 |
//@formatter:off
|
|
385 |
// from src/share/vm/oops/klass.hpp:
|
|
386 |
//
|
|
387 |
// For arrays, layout helper is a negative number, containing four
|
|
388 |
// distinct bytes, as follows:
|
|
389 |
// MSB:[tag, hsz, ebt, log2(esz)]:LSB
|
|
390 |
// where:
|
|
391 |
// tag is 0x80 if the elements are oops, 0xC0 if non-oops
|
|
392 |
// hsz is array header size in bytes (i.e., offset of first element)
|
|
393 |
// ebt is the BasicType of the elements
|
|
394 |
// esz is the element size in bytes
|
|
395 |
//@formatter:on
|
|
396 |
|
|
397 |
int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
|
|
398 |
int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
|
|
399 |
|
46344
|
400 |
Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, false, "dynamic type", true, options, counters);
|
46371
|
401 |
return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
|
43972
|
402 |
}
|
|
403 |
|
|
404 |
/**
|
|
405 |
* Calls the runtime stub for implementing MULTIANEWARRAY.
|
|
406 |
*/
|
|
407 |
@Snippet
|
|
408 |
public static Object newmultiarray(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
|
|
409 |
Word dims = DimensionsNode.allocaDimsArray(rank);
|
|
410 |
ExplodeLoopNode.explodeLoop();
|
|
411 |
for (int i = 0; i < rank; i++) {
|
46344
|
412 |
dims.writeInt(i * 4, dimensions[i], LocationIdentity.init());
|
43972
|
413 |
}
|
|
414 |
return newArrayCall(HotSpotBackend.NEW_MULTI_ARRAY, hub, rank, dims);
|
|
415 |
}
|
|
416 |
|
|
417 |
@Snippet
|
|
418 |
public static Object newmultiarrayPIC(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
|
46566
|
419 |
// Array type would be resolved by dominating resolution.
|
46509
|
420 |
KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
|
|
421 |
return newmultiarray(picHub, rank, dimensions);
|
43972
|
422 |
}
|
|
423 |
|
46371
|
424 |
@NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
|
43972
|
425 |
public static native Object newArrayCall(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims);
|
|
426 |
|
|
427 |
/**
|
|
428 |
* Maximum number of long stores to emit when zeroing an object with a constant size. Larger
|
|
429 |
* objects have their bodies initialized in a loop.
|
|
430 |
*/
|
|
431 |
private static final int MAX_UNROLLED_OBJECT_ZEROING_STORES = 8;
|
|
432 |
|
|
433 |
/**
|
|
434 |
* Zero uninitialized memory in a newly allocated object, unrolling as necessary and ensuring
|
|
435 |
* that stores are aligned.
|
|
436 |
*
|
|
437 |
* @param size number of bytes to zero
|
|
438 |
* @param memory beginning of object which is being zeroed
|
|
439 |
* @param constantSize is {@code size} known to be constant in the snippet
|
|
440 |
* @param startOffset offset to begin zeroing. May not be word aligned.
|
|
441 |
* @param manualUnroll maximally unroll zeroing
|
|
442 |
*/
|
46344
|
443 |
private static void zeroMemory(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
|
|
444 |
fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, counters);
|
43972
|
445 |
}
|
|
446 |
|
46344
|
447 |
private static void fillMemory(long value, int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
|
43972
|
448 |
ReplacementsUtil.runtimeAssert((size & 0x7) == 0, "unaligned object size");
|
|
449 |
int offset = startOffset;
|
|
450 |
if ((offset & 0x7) != 0) {
|
46344
|
451 |
memory.writeInt(offset, (int) value, LocationIdentity.init());
|
43972
|
452 |
offset += 4;
|
|
453 |
}
|
|
454 |
ReplacementsUtil.runtimeAssert((offset & 0x7) == 0, "unaligned offset");
|
|
455 |
if (manualUnroll && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
|
|
456 |
ReplacementsUtil.staticAssert(!constantSize, "size shouldn't be constant at instantiation time");
|
|
457 |
// This case handles arrays of constant length. Instead of having a snippet variant for
|
|
458 |
// each length, generate a chain of stores of maximum length. Once it's inlined the
|
|
459 |
// break statement will trim excess stores.
|
46344
|
460 |
if (counters != null && counters.instanceSeqInit != null) {
|
|
461 |
counters.instanceSeqInit.inc();
|
43972
|
462 |
}
|
46344
|
463 |
|
43972
|
464 |
explodeLoop();
|
|
465 |
for (int i = 0; i < MAX_UNROLLED_OBJECT_ZEROING_STORES; i++, offset += 8) {
|
|
466 |
if (offset == size) {
|
|
467 |
break;
|
|
468 |
}
|
46344
|
469 |
memory.initializeLong(offset, value, LocationIdentity.init());
|
43972
|
470 |
}
|
|
471 |
} else {
|
|
472 |
// Use Word instead of int to avoid extension to long in generated code
|
46459
|
473 |
Word off = WordFactory.signed(offset);
|
43972
|
474 |
if (constantSize && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
|
46344
|
475 |
if (counters != null && counters.instanceSeqInit != null) {
|
|
476 |
counters.instanceSeqInit.inc();
|
43972
|
477 |
}
|
|
478 |
explodeLoop();
|
|
479 |
} else {
|
46344
|
480 |
if (counters != null && counters.instanceLoopInit != null) {
|
|
481 |
counters.instanceLoopInit.inc();
|
43972
|
482 |
}
|
|
483 |
}
|
|
484 |
for (; off.rawValue() < size; off = off.add(8)) {
|
46344
|
485 |
memory.initializeLong(off, value, LocationIdentity.init());
|
43972
|
486 |
}
|
|
487 |
}
|
|
488 |
}
|
|
489 |
|
|
490 |
/**
|
|
491 |
* Fill uninitialized memory with garbage value in a newly allocated object, unrolling as
|
|
492 |
* necessary and ensuring that stores are aligned.
|
|
493 |
*
|
|
494 |
* @param size number of bytes to zero
|
|
495 |
* @param memory beginning of object which is being zeroed
|
|
496 |
* @param constantSize is {@code size} known to be constant in the snippet
|
|
497 |
* @param startOffset offset to begin zeroing. May not be word aligned.
|
|
498 |
* @param manualUnroll maximally unroll zeroing
|
|
499 |
*/
|
46344
|
500 |
private static void fillWithGarbage(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
|
|
501 |
fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, counters);
|
43972
|
502 |
}
|
|
503 |
|
|
504 |
/**
|
|
505 |
* Formats some allocated memory with an object header and zeroes out the rest. Disables asserts
|
|
506 |
* since they can't be compiled in stubs.
|
|
507 |
*/
|
|
508 |
public static Object formatObjectForStub(KlassPointer hub, int size, Word memory, Word compileTimePrototypeMarkWord) {
|
46344
|
509 |
return formatObject(hub, size, memory, compileTimePrototypeMarkWord, true, false, null);
|
43972
|
510 |
}
|
|
511 |
|
|
512 |
/**
|
|
513 |
* Formats some allocated memory with an object header and zeroes out the rest.
|
|
514 |
*/
|
46344
|
515 |
protected static Object formatObject(KlassPointer hub, int size, Word memory, Word compileTimePrototypeMarkWord, boolean fillContents, boolean constantSize, Counters counters) {
|
43972
|
516 |
Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord;
|
|
517 |
initializeObjectHeader(memory, prototypeMarkWord, hub);
|
|
518 |
if (fillContents) {
|
46344
|
519 |
zeroMemory(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
|
43972
|
520 |
} else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
|
46344
|
521 |
fillWithGarbage(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
|
43972
|
522 |
}
|
46344
|
523 |
MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
|
|
524 |
return memory.toObjectNonNull();
|
43972
|
525 |
}
|
|
526 |
|
|
527 |
@Snippet
|
|
528 |
protected static void verifyHeap(@ConstantParameter Register threadRegister) {
|
|
529 |
Word thread = registerAsWord(threadRegister);
|
|
530 |
Word topValue = readTlabTop(thread);
|
46459
|
531 |
if (!topValue.equal(WordFactory.zero())) {
|
43972
|
532 |
Word topValueContents = topValue.readWord(0, MARK_WORD_LOCATION);
|
46459
|
533 |
if (topValueContents.equal(WordFactory.zero())) {
|
43972
|
534 |
AssertionSnippets.vmMessageC(AssertionSnippets.ASSERTION_VM_MESSAGE_C, true, cstring("overzeroing of TLAB detected"), 0L, 0L, 0L);
|
|
535 |
}
|
|
536 |
}
|
|
537 |
}
|
|
538 |
|
|
539 |
/**
|
|
540 |
* Formats some allocated memory with an object header and zeroes out the rest.
|
|
541 |
*/
|
|
542 |
public static Object formatArray(KlassPointer hub, int allocationSize, int length, int headerSize, Word memory, Word prototypeMarkWord, boolean fillContents, boolean maybeUnroll,
|
46344
|
543 |
Counters counters) {
|
|
544 |
memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, LocationIdentity.init());
|
43972
|
545 |
/*
|
|
546 |
* store hub last as the concurrent garbage collectors assume length is valid if hub field
|
|
547 |
* is not null
|
|
548 |
*/
|
|
549 |
initializeObjectHeader(memory, prototypeMarkWord, hub);
|
|
550 |
if (fillContents) {
|
46344
|
551 |
zeroMemory(allocationSize, memory, false, headerSize, maybeUnroll, counters);
|
43972
|
552 |
} else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
|
46344
|
553 |
fillWithGarbage(allocationSize, memory, false, headerSize, maybeUnroll, counters);
|
43972
|
554 |
}
|
46344
|
555 |
MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
|
|
556 |
return memory.toObjectNonNull();
|
|
557 |
}
|
|
558 |
|
|
559 |
static class Counters {
|
|
560 |
Counters(SnippetCounter.Group.Factory factory) {
|
|
561 |
Group newInstance = factory.createSnippetCounterGroup("NewInstance");
|
|
562 |
Group newArray = factory.createSnippetCounterGroup("NewArray");
|
|
563 |
instanceSeqInit = new SnippetCounter(newInstance, "tlabSeqInit", "TLAB alloc with unrolled zeroing");
|
|
564 |
instanceLoopInit = new SnippetCounter(newInstance, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
|
|
565 |
arrayLoopInit = new SnippetCounter(newArray, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
|
|
566 |
stub = new SnippetCounter(newInstance, "stub", "alloc and zeroing via stub");
|
|
567 |
}
|
|
568 |
|
|
569 |
final SnippetCounter instanceSeqInit;
|
|
570 |
final SnippetCounter instanceLoopInit;
|
|
571 |
final SnippetCounter arrayLoopInit;
|
|
572 |
final SnippetCounter stub;
|
43972
|
573 |
}
|
|
574 |
|
|
575 |
public static class Templates extends AbstractTemplates {
|
|
576 |
|
46344
|
577 |
private final SnippetInfo allocateInstance = snippet(NewObjectSnippets.class, "allocateInstance", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
|
|
578 |
private final SnippetInfo allocateInstancePIC = snippet(NewObjectSnippets.class, "allocateInstancePIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
|
43972
|
579 |
TLAB_END_LOCATION);
|
46344
|
580 |
private final SnippetInfo allocateArray = snippet(NewObjectSnippets.class, "allocateArray", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
|
|
581 |
private final SnippetInfo allocateArrayPIC = snippet(NewObjectSnippets.class, "allocateArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
|
|
582 |
private final SnippetInfo allocatePrimitiveArrayPIC = snippet(NewObjectSnippets.class, "allocatePrimitiveArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
|
43972
|
583 |
TLAB_END_LOCATION);
|
46344
|
584 |
private final SnippetInfo allocateArrayDynamic = snippet(NewObjectSnippets.class, "allocateArrayDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
|
43972
|
585 |
TLAB_END_LOCATION);
|
46344
|
586 |
private final SnippetInfo allocateInstanceDynamic = snippet(NewObjectSnippets.class, "allocateInstanceDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
|
43972
|
587 |
TLAB_END_LOCATION);
|
46344
|
588 |
private final SnippetInfo newmultiarray = snippet(NewObjectSnippets.class, "newmultiarray", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
|
|
589 |
private final SnippetInfo newmultiarrayPIC = snippet(NewObjectSnippets.class, "newmultiarrayPIC", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
|
43972
|
590 |
private final SnippetInfo verifyHeap = snippet(NewObjectSnippets.class, "verifyHeap");
|
|
591 |
private final GraalHotSpotVMConfig config;
|
46344
|
592 |
private final Counters counters;
|
43972
|
593 |
|
46640
|
594 |
public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target,
|
|
595 |
GraalHotSpotVMConfig config) {
|
|
596 |
super(options, factories, providers, providers.getSnippetReflection(), target);
|
43972
|
597 |
this.config = config;
|
46344
|
598 |
counters = new Counters(factory);
|
43972
|
599 |
}
|
|
600 |
|
|
601 |
/**
|
|
602 |
* Lowers a {@link NewInstanceNode}.
|
|
603 |
*/
|
|
604 |
public void lower(NewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
|
|
605 |
StructuredGraph graph = newInstanceNode.graph();
|
|
606 |
HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newInstanceNode.instanceClass();
|
|
607 |
assert !type.isArray();
|
|
608 |
ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
|
|
609 |
int size = instanceSize(type);
|
|
610 |
|
46344
|
611 |
OptionValues localOptions = graph.getOptions();
|
|
612 |
SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? allocateInstancePIC : allocateInstance;
|
43972
|
613 |
Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
|
|
614 |
args.addConst("size", size);
|
|
615 |
args.add("hub", hub);
|
|
616 |
args.add("prototypeMarkWord", type.prototypeMarkWord());
|
|
617 |
args.addConst("fillContents", newInstanceNode.fillContents());
|
|
618 |
args.addConst("threadRegister", registers.getThreadRegister());
|
|
619 |
args.addConst("constantSize", true);
|
46344
|
620 |
args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? type.toJavaName(false) : "");
|
|
621 |
args.addConst("options", localOptions);
|
|
622 |
args.addConst("counters", counters);
|
43972
|
623 |
|
46640
|
624 |
SnippetTemplate template = template(graph.getDebug(), args);
|
|
625 |
graph.getDebug().log("Lowering allocateInstance in %s: node=%s, template=%s, arguments=%s", graph, newInstanceNode, template, args);
|
43972
|
626 |
template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
|
|
627 |
}
|
|
628 |
|
|
629 |
/**
|
|
630 |
* Lowers a {@link NewArrayNode}.
|
|
631 |
*/
|
|
632 |
public void lower(NewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
|
|
633 |
StructuredGraph graph = newArrayNode.graph();
|
|
634 |
ResolvedJavaType elementType = newArrayNode.elementType();
|
|
635 |
HotSpotResolvedObjectType arrayType = (HotSpotResolvedObjectType) elementType.getArrayClass();
|
|
636 |
JavaKind elementKind = elementType.getJavaKind();
|
|
637 |
ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayType.klass(), providers.getMetaAccess(), graph);
|
|
638 |
final int headerSize = getArrayBaseOffset(elementKind);
|
|
639 |
int log2ElementSize = CodeUtil.log2(HotSpotJVMCIRuntimeProvider.getArrayIndexScale(elementKind));
|
|
640 |
|
46344
|
641 |
OptionValues localOptions = graph.getOptions();
|
43972
|
642 |
SnippetInfo snippet;
|
46344
|
643 |
if (GeneratePIC.getValue(localOptions)) {
|
43972
|
644 |
if (elementType.isPrimitive()) {
|
|
645 |
snippet = allocatePrimitiveArrayPIC;
|
|
646 |
} else {
|
|
647 |
snippet = allocateArrayPIC;
|
|
648 |
}
|
|
649 |
} else {
|
|
650 |
snippet = allocateArray;
|
|
651 |
}
|
|
652 |
|
|
653 |
Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
|
|
654 |
args.add("hub", hub);
|
|
655 |
ValueNode length = newArrayNode.length();
|
|
656 |
args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
|
|
657 |
assert arrayType.prototypeMarkWord() == lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord() : "all array types are assumed to have the same prototypeMarkWord";
|
|
658 |
args.add("prototypeMarkWord", arrayType.prototypeMarkWord());
|
|
659 |
args.addConst("headerSize", headerSize);
|
|
660 |
args.addConst("log2ElementSize", log2ElementSize);
|
|
661 |
args.addConst("fillContents", newArrayNode.fillContents());
|
|
662 |
args.addConst("threadRegister", registers.getThreadRegister());
|
|
663 |
args.addConst("maybeUnroll", length.isConstant());
|
46344
|
664 |
args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? arrayType.toJavaName(false) : "");
|
|
665 |
args.addConst("options", localOptions);
|
|
666 |
args.addConst("counters", counters);
|
46640
|
667 |
SnippetTemplate template = template(graph.getDebug(), args);
|
|
668 |
graph.getDebug().log("Lowering allocateArray in %s: node=%s, template=%s, arguments=%s", graph, newArrayNode, template, args);
|
43972
|
669 |
template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
|
|
670 |
}
|
|
671 |
|
|
672 |
public void lower(DynamicNewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
|
|
673 |
Arguments args = new Arguments(allocateInstanceDynamic, newInstanceNode.graph().getGuardsStage(), tool.getLoweringStage());
|
46344
|
674 |
OptionValues localOptions = newInstanceNode.getOptions();
|
43972
|
675 |
args.add("type", newInstanceNode.getInstanceType());
|
|
676 |
ValueNode classClass = newInstanceNode.getClassClass();
|
|
677 |
assert classClass != null;
|
|
678 |
args.add("classClass", classClass);
|
|
679 |
args.addConst("fillContents", newInstanceNode.fillContents());
|
|
680 |
args.addConst("threadRegister", registers.getThreadRegister());
|
46344
|
681 |
args.addConst("options", localOptions);
|
|
682 |
args.addConst("counters", counters);
|
43972
|
683 |
|
46640
|
684 |
SnippetTemplate template = template(newInstanceNode.getDebug(), args);
|
43972
|
685 |
template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
|
|
686 |
}
|
|
687 |
|
|
688 |
public void lower(DynamicNewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
|
|
689 |
StructuredGraph graph = newArrayNode.graph();
|
46344
|
690 |
OptionValues localOptions = graph.getOptions();
|
43972
|
691 |
Arguments args = new Arguments(allocateArrayDynamic, newArrayNode.graph().getGuardsStage(), tool.getLoweringStage());
|
|
692 |
args.add("elementType", newArrayNode.getElementType());
|
|
693 |
ValueNode voidClass = newArrayNode.getVoidClass();
|
|
694 |
assert voidClass != null;
|
|
695 |
args.add("voidClass", voidClass);
|
|
696 |
ValueNode length = newArrayNode.length();
|
|
697 |
args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
|
|
698 |
args.addConst("fillContents", newArrayNode.fillContents());
|
|
699 |
args.addConst("threadRegister", registers.getThreadRegister());
|
|
700 |
/*
|
|
701 |
* We use Kind.Illegal as a marker value instead of null because constant snippet
|
|
702 |
* parameters cannot be null.
|
|
703 |
*/
|
|
704 |
args.addConst("knownElementKind", newArrayNode.getKnownElementKind() == null ? JavaKind.Illegal : newArrayNode.getKnownElementKind());
|
|
705 |
if (newArrayNode.getKnownElementKind() != null) {
|
|
706 |
args.addConst("knownLayoutHelper", lookupArrayClass(tool, newArrayNode.getKnownElementKind()).layoutHelper());
|
|
707 |
} else {
|
|
708 |
args.addConst("knownLayoutHelper", 0);
|
|
709 |
}
|
|
710 |
args.add("prototypeMarkWord", lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord());
|
46344
|
711 |
args.addConst("options", localOptions);
|
|
712 |
args.addConst("counters", counters);
|
46640
|
713 |
SnippetTemplate template = template(graph.getDebug(), args);
|
43972
|
714 |
template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
|
|
715 |
}
|
|
716 |
|
|
717 |
private static HotSpotResolvedObjectType lookupArrayClass(LoweringTool tool, JavaKind kind) {
|
|
718 |
return (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(kind == JavaKind.Object ? Object.class : kind.toJavaClass()).getArrayClass();
|
|
719 |
}
|
|
720 |
|
|
721 |
public void lower(NewMultiArrayNode newmultiarrayNode, LoweringTool tool) {
|
|
722 |
StructuredGraph graph = newmultiarrayNode.graph();
|
46344
|
723 |
OptionValues localOptions = graph.getOptions();
|
43972
|
724 |
int rank = newmultiarrayNode.dimensionCount();
|
|
725 |
ValueNode[] dims = new ValueNode[rank];
|
|
726 |
for (int i = 0; i < newmultiarrayNode.dimensionCount(); i++) {
|
|
727 |
dims[i] = newmultiarrayNode.dimension(i);
|
|
728 |
}
|
|
729 |
HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newmultiarrayNode.type();
|
|
730 |
ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
|
|
731 |
|
46344
|
732 |
SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? newmultiarrayPIC : newmultiarray;
|
43972
|
733 |
Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
|
|
734 |
args.add("hub", hub);
|
|
735 |
args.addConst("rank", rank);
|
|
736 |
args.addVarargs("dimensions", int.class, StampFactory.forKind(JavaKind.Int), dims);
|
46640
|
737 |
template(newmultiarrayNode.getDebug(), args).instantiate(providers.getMetaAccess(), newmultiarrayNode, DEFAULT_REPLACER, args);
|
43972
|
738 |
}
|
|
739 |
|
|
740 |
private static int instanceSize(HotSpotResolvedObjectType type) {
|
|
741 |
int size = type.instanceSize();
|
|
742 |
assert size >= 0;
|
|
743 |
return size;
|
|
744 |
}
|
|
745 |
|
|
746 |
public void lower(VerifyHeapNode verifyHeapNode, HotSpotRegistersProvider registers, LoweringTool tool) {
|
|
747 |
if (config.cAssertions) {
|
|
748 |
Arguments args = new Arguments(verifyHeap, verifyHeapNode.graph().getGuardsStage(), tool.getLoweringStage());
|
|
749 |
args.addConst("threadRegister", registers.getThreadRegister());
|
|
750 |
|
46640
|
751 |
SnippetTemplate template = template(verifyHeapNode.getDebug(), args);
|
43972
|
752 |
template.instantiate(providers.getMetaAccess(), verifyHeapNode, DEFAULT_REPLACER, args);
|
|
753 |
} else {
|
|
754 |
GraphUtil.removeFixedWithUnusedInputs(verifyHeapNode);
|
|
755 |
}
|
|
756 |
}
|
|
757 |
}
|
|
758 |
}
|