--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/NewObjectSnippets.java Tue Sep 12 19:03:39 2017 +0200
@@ -0,0 +1,758 @@
+/*
+ * Copyright (c) 2012, 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.compiler.hotspot.replacements;
+
+import static jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider.getArrayBaseOffset;
+import static jdk.vm.ci.hotspot.HotSpotMetaAccessProvider.computeArrayAllocationSize;
+import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
+import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan;
+import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.config;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
+import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocations;
+import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocationsContext;
+import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCastToSnippetReplaceeStamp;
+import static org.graalvm.compiler.nodes.PiNode.piCastToSnippetReplaceeStamp;
+import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
+import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
+import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
+import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
+import static org.graalvm.compiler.replacements.ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED;
+import static org.graalvm.compiler.replacements.ReplacementsUtil.staticAssert;
+import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
+import static org.graalvm.compiler.replacements.nodes.CStringConstant.cstring;
+import static org.graalvm.compiler.replacements.nodes.ExplodeLoopNode.explodeLoop;
+
+import org.graalvm.compiler.api.replacements.Fold;
+import org.graalvm.compiler.api.replacements.Snippet;
+import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
+import org.graalvm.compiler.api.replacements.Snippet.VarargsParameter;
+import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
+import org.graalvm.compiler.core.common.type.StampFactory;
+import org.graalvm.compiler.debug.DebugHandlersFactory;
+import org.graalvm.compiler.debug.GraalError;
+import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
+import org.graalvm.compiler.graph.Node.NodeIntrinsic;
+import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
+import org.graalvm.compiler.hotspot.HotSpotBackend;
+import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
+import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
+import org.graalvm.compiler.hotspot.nodes.DimensionsNode;
+import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyFixedNode;
+import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyNode;
+import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
+import org.graalvm.compiler.hotspot.word.KlassPointer;
+import org.graalvm.compiler.nodes.ConstantNode;
+import org.graalvm.compiler.nodes.DeoptimizeNode;
+import org.graalvm.compiler.nodes.PiNode;
+import org.graalvm.compiler.nodes.PrefetchAllocateNode;
+import org.graalvm.compiler.nodes.SnippetAnchorNode;
+import org.graalvm.compiler.nodes.StructuredGraph;
+import org.graalvm.compiler.nodes.ValueNode;
+import org.graalvm.compiler.nodes.debug.DynamicCounterNode;
+import org.graalvm.compiler.nodes.debug.VerifyHeapNode;
+import org.graalvm.compiler.nodes.extended.ForeignCallNode;
+import org.graalvm.compiler.nodes.extended.MembarNode;
+import org.graalvm.compiler.nodes.java.DynamicNewArrayNode;
+import org.graalvm.compiler.nodes.java.DynamicNewInstanceNode;
+import org.graalvm.compiler.nodes.java.NewArrayNode;
+import org.graalvm.compiler.nodes.java.NewInstanceNode;
+import org.graalvm.compiler.nodes.java.NewMultiArrayNode;
+import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
+import org.graalvm.compiler.nodes.spi.LoweringTool;
+import org.graalvm.compiler.nodes.util.GraphUtil;
+import org.graalvm.compiler.options.OptionValues;
+import org.graalvm.compiler.replacements.ReplacementsUtil;
+import org.graalvm.compiler.replacements.SnippetCounter;
+import org.graalvm.compiler.replacements.SnippetCounter.Group;
+import org.graalvm.compiler.replacements.SnippetTemplate;
+import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
+import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
+import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
+import org.graalvm.compiler.replacements.Snippets;
+import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode;
+import org.graalvm.compiler.word.Word;
+import org.graalvm.word.LocationIdentity;
+import org.graalvm.word.WordFactory;
+
+import jdk.vm.ci.code.CodeUtil;
+import jdk.vm.ci.code.MemoryBarriers;
+import jdk.vm.ci.code.Register;
+import jdk.vm.ci.code.TargetDescription;
+import jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider;
+import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
+import jdk.vm.ci.meta.DeoptimizationAction;
+import jdk.vm.ci.meta.DeoptimizationReason;
+import jdk.vm.ci.meta.JavaKind;
+import jdk.vm.ci.meta.ResolvedJavaType;
+
+/**
+ * Snippets used for implementing NEW, ANEWARRAY and NEWARRAY.
+ */
+public class NewObjectSnippets implements Snippets {
+
+ enum ProfileContext {
+ AllocatingMethod,
+ InstanceOrArray,
+ AllocatedType,
+ AllocatedTypesInMethod,
+ Total
+ }
+
+ @Fold
+ static String createName(String path, String typeContext, OptionValues options) {
+ switch (ProfileAllocationsContext.getValue(options)) {
+ case AllocatingMethod:
+ return "";
+ case InstanceOrArray:
+ return path;
+ case AllocatedType:
+ case AllocatedTypesInMethod:
+ return typeContext;
+ case Total:
+ return "bytes";
+ default:
+ throw GraalError.shouldNotReachHere();
+ }
+ }
+
+ @Fold
+ static boolean doProfile(OptionValues options) {
+ return ProfileAllocations.getValue(options);
+ }
+
+ @Fold
+ static boolean withContext(OptionValues options) {
+ ProfileContext context = ProfileAllocationsContext.getValue(options);
+ return context == ProfileContext.AllocatingMethod || context == ProfileContext.AllocatedTypesInMethod;
+ }
+
+ protected static void profileAllocation(String path, long size, String typeContext, OptionValues options) {
+ if (doProfile(options)) {
+ String name = createName(path, typeContext, options);
+
+ boolean context = withContext(options);
+ DynamicCounterNode.counter(name, "number of bytes allocated", size, context);
+ DynamicCounterNode.counter(name, "number of allocations", 1, context);
+ }
+ }
+
+ public static void emitPrefetchAllocate(Word address, boolean isArray) {
+ GraalHotSpotVMConfig config = config(INJECTED_VMCONFIG);
+ if (config.allocatePrefetchStyle > 0) {
+ // Insert a prefetch for each allocation only on the fast-path
+ // Generate several prefetch instructions.
+ int lines = isArray ? config.allocatePrefetchLines : config.allocateInstancePrefetchLines;
+ int stepSize = config.allocatePrefetchStepSize;
+ int distance = config.allocatePrefetchDistance;
+ ExplodeLoopNode.explodeLoop();
+ for (int i = 0; i < lines; i++) {
+ PrefetchAllocateNode.prefetch(OffsetAddressNode.address(address, distance));
+ distance += stepSize;
+ }
+ }
+ }
+
+ @Snippet
+ public static Object allocateInstance(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
+ @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter OptionValues options,
+ @ConstantParameter Counters counters) {
+ return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, options, counters));
+ }
+
+ public static Object allocateInstanceHelper(int size, KlassPointer hub, Word prototypeMarkWord, boolean fillContents,
+ Register threadRegister, boolean constantSize, String typeContext, OptionValues options, Counters counters) {
+ Object result;
+ Word thread = registerAsWord(threadRegister);
+ Word top = readTlabTop(thread);
+ Word end = readTlabEnd(thread);
+ Word newTop = top.add(size);
+ if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
+ writeTlabTop(thread, newTop);
+ emitPrefetchAllocate(newTop, false);
+ result = formatObject(hub, size, top, prototypeMarkWord, fillContents, constantSize, counters);
+ } else {
+ if (counters != null && counters.stub != null) {
+ counters.stub.inc();
+ }
+ result = newInstance(HotSpotBackend.NEW_INSTANCE, hub);
+ }
+ profileAllocation("instance", size, typeContext, options);
+ return verifyOop(result);
+ }
+
+ @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
+ public static native Object newInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
+
+ @Snippet
+ public static Object allocateInstancePIC(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
+ @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter OptionValues options,
+ @ConstantParameter Counters counters) {
+ // Klass must be initialized by the time the first instance is allocated, therefore we can
+ // just load it from the corresponding cell and avoid the resolution check. We have to use a
+ // fixed load though, to prevent it from floating above the initialization.
+ KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
+ return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, picHub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, options, counters));
+ }
+
+ @Snippet
+ public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
+ @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
+ if (probability(SLOW_PATH_PROBABILITY, type == null)) {
+ DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
+ }
+ Class<?> nonNullType = PiNode.piCastNonNullClass(type, SnippetAnchorNode.anchor());
+
+ if (probability(SLOW_PATH_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) {
+ DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
+ }
+
+ return PiNode.piCastToSnippetReplaceeStamp(allocateInstanceDynamicHelper(type, fillContents, threadRegister, options, counters, nonNullType));
+ }
+
+ private static Object allocateInstanceDynamicHelper(Class<?> type, boolean fillContents, Register threadRegister, OptionValues options, Counters counters, Class<?> nonNullType) {
+ KlassPointer hub = ClassGetHubNode.readClass(nonNullType);
+ if (probability(FAST_PATH_PROBABILITY, !hub.isNull())) {
+ KlassPointer nonNullHub = ClassGetHubNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
+
+ if (probability(FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(nonNullHub))) {
+ int layoutHelper = readLayoutHelper(nonNullHub);
+ /*
+ * src/share/vm/oops/klass.hpp: For instances, layout helper is a positive number,
+ * the instance size. This size is already passed through align_object_size and
+ * scaled to bytes. The low order bit is set if instances of this class cannot be
+ * allocated using the fastpath.
+ */
+ if (probability(FAST_PATH_PROBABILITY, (layoutHelper & 1) == 0)) {
+ Word prototypeMarkWord = nonNullHub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
+ /*
+ * FIXME(je,ds): we should actually pass typeContext instead of "" but late
+ * binding of parameters is not yet supported by the GraphBuilderPlugin system.
+ */
+ return allocateInstanceHelper(layoutHelper, nonNullHub, prototypeMarkWord, fillContents, threadRegister, false, "", options, counters);
+ }
+ }
+ }
+ return dynamicNewInstanceStub(type);
+ }
+
+ /**
+ * Maximum array length for which fast path allocation is used.
+ */
+ public static final int MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH = 0x00FFFFFF;
+
+ @Snippet
+ public static Object allocatePrimitiveArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
+ @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
+ @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
+ // Primitive array types are eagerly pre-resolved. We can use a floating load.
+ KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
+ return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
+ }
+
+ @Snippet
+ public static Object allocateArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
+ @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
+ @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
+ // Array type would be resolved by dominating resolution.
+ KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
+ return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
+ }
+
+ @Snippet
+ public static Object allocateArray(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
+ @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
+ @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
+ Object result = allocateArrayImpl(hub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
+ return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
+ }
+
+ private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister,
+ boolean maybeUnroll, String typeContext, boolean skipNegativeCheck, OptionValues options, Counters counters) {
+ Object result;
+ int alignment = wordSize();
+ int allocationSize = computeArrayAllocationSize(length, alignment, headerSize, log2ElementSize);
+ Word thread = registerAsWord(threadRegister);
+ Word top = readTlabTop(thread);
+ Word end = readTlabEnd(thread);
+ Word newTop = top.add(allocationSize);
+ if (probability(FREQUENT_PROBABILITY, skipNegativeCheck || belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&
+ probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
+ writeTlabTop(thread, newTop);
+ emitPrefetchAllocate(newTop, true);
+ if (counters != null && counters.arrayLoopInit != null) {
+ counters.arrayLoopInit.inc();
+ }
+ result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, maybeUnroll, counters);
+ } else {
+ result = newArray(HotSpotBackend.NEW_ARRAY, hub, length, fillContents);
+ }
+ profileAllocation("array", allocationSize, typeContext, options);
+ return result;
+ }
+
+ @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
+ public static native Object newArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length, boolean fillContents);
+
+ public static final ForeignCallDescriptor DYNAMIC_NEW_ARRAY = new ForeignCallDescriptor("dynamic_new_array", Object.class, Class.class, int.class);
+ public static final ForeignCallDescriptor DYNAMIC_NEW_INSTANCE = new ForeignCallDescriptor("dynamic_new_instance", Object.class, Class.class);
+
+ @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
+ public static native Object dynamicNewArrayStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType, int length);
+
+ public static Object dynamicNewInstanceStub(Class<?> elementType) {
+ return dynamicNewInstanceStubCall(DYNAMIC_NEW_INSTANCE, elementType);
+ }
+
+ @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
+ public static native Object dynamicNewInstanceStubCall(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
+
+ @Snippet
+ public static Object allocateArrayDynamic(Class<?> elementType, Class<?> voidClass, int length, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
+ @ConstantParameter JavaKind knownElementKind, @ConstantParameter int knownLayoutHelper, Word prototypeMarkWord, @ConstantParameter OptionValues options,
+ @ConstantParameter Counters counters) {
+ Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, threadRegister, knownElementKind, knownLayoutHelper, prototypeMarkWord, options, counters);
+ return result;
+ }
+
+ private static Object allocateArrayDynamicImpl(Class<?> elementType, Class<?> voidClass, int length, boolean fillContents, Register threadRegister, JavaKind knownElementKind,
+ int knownLayoutHelper, Word prototypeMarkWord, OptionValues options, Counters counters) {
+ /*
+ * We only need the dynamic check for void when we have no static information from
+ * knownElementKind.
+ */
+ staticAssert(knownElementKind != JavaKind.Void, "unsupported knownElementKind");
+ if (knownElementKind == JavaKind.Illegal && probability(SLOW_PATH_PROBABILITY, elementType == null || DynamicNewArrayNode.throwsIllegalArgumentException(elementType, voidClass))) {
+ DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
+ }
+
+ KlassPointer klass = loadKlassFromObject(elementType, arrayKlassOffset(INJECTED_VMCONFIG), CLASS_ARRAY_KLASS_LOCATION);
+ if (klass.isNull()) {
+ DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
+ }
+ KlassPointer nonNullKlass = ClassGetHubNode.piCastNonNull(klass, SnippetAnchorNode.anchor());
+
+ if (length < 0) {
+ DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
+ }
+ int layoutHelper = knownElementKind != JavaKind.Illegal ? knownLayoutHelper : readLayoutHelper(nonNullKlass);
+ //@formatter:off
+ // from src/share/vm/oops/klass.hpp:
+ //
+ // For arrays, layout helper is a negative number, containing four
+ // distinct bytes, as follows:
+ // MSB:[tag, hsz, ebt, log2(esz)]:LSB
+ // where:
+ // tag is 0x80 if the elements are oops, 0xC0 if non-oops
+ // hsz is array header size in bytes (i.e., offset of first element)
+ // ebt is the BasicType of the elements
+ // esz is the element size in bytes
+ //@formatter:on
+
+ int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
+ int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
+
+ Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, false, "dynamic type", true, options, counters);
+ return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
+ }
+
+ /**
+ * Calls the runtime stub for implementing MULTIANEWARRAY.
+ */
+ @Snippet
+ public static Object newmultiarray(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
+ Word dims = DimensionsNode.allocaDimsArray(rank);
+ ExplodeLoopNode.explodeLoop();
+ for (int i = 0; i < rank; i++) {
+ dims.writeInt(i * 4, dimensions[i], LocationIdentity.init());
+ }
+ return newArrayCall(HotSpotBackend.NEW_MULTI_ARRAY, hub, rank, dims);
+ }
+
+ @Snippet
+ public static Object newmultiarrayPIC(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
+ // Array type would be resolved by dominating resolution.
+ KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
+ return newmultiarray(picHub, rank, dimensions);
+ }
+
+ @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
+ public static native Object newArrayCall(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims);
+
+ /**
+ * Maximum number of long stores to emit when zeroing an object with a constant size. Larger
+ * objects have their bodies initialized in a loop.
+ */
+ private static final int MAX_UNROLLED_OBJECT_ZEROING_STORES = 8;
+
+ /**
+ * Zero uninitialized memory in a newly allocated object, unrolling as necessary and ensuring
+ * that stores are aligned.
+ *
+ * @param size number of bytes to zero
+ * @param memory beginning of object which is being zeroed
+ * @param constantSize is {@code size} known to be constant in the snippet
+ * @param startOffset offset to begin zeroing. May not be word aligned.
+ * @param manualUnroll maximally unroll zeroing
+ */
+ private static void zeroMemory(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
+ fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, counters);
+ }
+
+ private static void fillMemory(long value, int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
+ ReplacementsUtil.runtimeAssert((size & 0x7) == 0, "unaligned object size");
+ int offset = startOffset;
+ if ((offset & 0x7) != 0) {
+ memory.writeInt(offset, (int) value, LocationIdentity.init());
+ offset += 4;
+ }
+ ReplacementsUtil.runtimeAssert((offset & 0x7) == 0, "unaligned offset");
+ if (manualUnroll && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
+ ReplacementsUtil.staticAssert(!constantSize, "size shouldn't be constant at instantiation time");
+ // This case handles arrays of constant length. Instead of having a snippet variant for
+ // each length, generate a chain of stores of maximum length. Once it's inlined the
+ // break statement will trim excess stores.
+ if (counters != null && counters.instanceSeqInit != null) {
+ counters.instanceSeqInit.inc();
+ }
+
+ explodeLoop();
+ for (int i = 0; i < MAX_UNROLLED_OBJECT_ZEROING_STORES; i++, offset += 8) {
+ if (offset == size) {
+ break;
+ }
+ memory.initializeLong(offset, value, LocationIdentity.init());
+ }
+ } else {
+ // Use Word instead of int to avoid extension to long in generated code
+ Word off = WordFactory.signed(offset);
+ if (constantSize && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
+ if (counters != null && counters.instanceSeqInit != null) {
+ counters.instanceSeqInit.inc();
+ }
+ explodeLoop();
+ } else {
+ if (counters != null && counters.instanceLoopInit != null) {
+ counters.instanceLoopInit.inc();
+ }
+ }
+ for (; off.rawValue() < size; off = off.add(8)) {
+ memory.initializeLong(off, value, LocationIdentity.init());
+ }
+ }
+ }
+
+ /**
+ * Fill uninitialized memory with garbage value in a newly allocated object, unrolling as
+ * necessary and ensuring that stores are aligned.
+ *
+ * @param size number of bytes to zero
+ * @param memory beginning of object which is being zeroed
+ * @param constantSize is {@code size} known to be constant in the snippet
+ * @param startOffset offset to begin zeroing. May not be word aligned.
+ * @param manualUnroll maximally unroll zeroing
+ */
+ private static void fillWithGarbage(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
+ fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, counters);
+ }
+
+ /**
+ * Formats some allocated memory with an object header and zeroes out the rest. Disables asserts
+ * since they can't be compiled in stubs.
+ */
+ public static Object formatObjectForStub(KlassPointer hub, int size, Word memory, Word compileTimePrototypeMarkWord) {
+ return formatObject(hub, size, memory, compileTimePrototypeMarkWord, true, false, null);
+ }
+
+ /**
+ * Formats some allocated memory with an object header and zeroes out the rest.
+ */
+ protected static Object formatObject(KlassPointer hub, int size, Word memory, Word compileTimePrototypeMarkWord, boolean fillContents, boolean constantSize, Counters counters) {
+ Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord;
+ initializeObjectHeader(memory, prototypeMarkWord, hub);
+ if (fillContents) {
+ zeroMemory(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
+ } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
+ fillWithGarbage(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
+ }
+ MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
+ return memory.toObjectNonNull();
+ }
+
+ @Snippet
+ protected static void verifyHeap(@ConstantParameter Register threadRegister) {
+ Word thread = registerAsWord(threadRegister);
+ Word topValue = readTlabTop(thread);
+ if (!topValue.equal(WordFactory.zero())) {
+ Word topValueContents = topValue.readWord(0, MARK_WORD_LOCATION);
+ if (topValueContents.equal(WordFactory.zero())) {
+ AssertionSnippets.vmMessageC(AssertionSnippets.ASSERTION_VM_MESSAGE_C, true, cstring("overzeroing of TLAB detected"), 0L, 0L, 0L);
+ }
+ }
+ }
+
+ /**
+ * Formats some allocated memory with an object header and zeroes out the rest.
+ */
+ public static Object formatArray(KlassPointer hub, int allocationSize, int length, int headerSize, Word memory, Word prototypeMarkWord, boolean fillContents, boolean maybeUnroll,
+ Counters counters) {
+ memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, LocationIdentity.init());
+ /*
+ * store hub last as the concurrent garbage collectors assume length is valid if hub field
+ * is not null
+ */
+ initializeObjectHeader(memory, prototypeMarkWord, hub);
+ if (fillContents) {
+ zeroMemory(allocationSize, memory, false, headerSize, maybeUnroll, counters);
+ } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
+ fillWithGarbage(allocationSize, memory, false, headerSize, maybeUnroll, counters);
+ }
+ MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
+ return memory.toObjectNonNull();
+ }
+
+ static class Counters {
+ Counters(SnippetCounter.Group.Factory factory) {
+ Group newInstance = factory.createSnippetCounterGroup("NewInstance");
+ Group newArray = factory.createSnippetCounterGroup("NewArray");
+ instanceSeqInit = new SnippetCounter(newInstance, "tlabSeqInit", "TLAB alloc with unrolled zeroing");
+ instanceLoopInit = new SnippetCounter(newInstance, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
+ arrayLoopInit = new SnippetCounter(newArray, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
+ stub = new SnippetCounter(newInstance, "stub", "alloc and zeroing via stub");
+ }
+
+ final SnippetCounter instanceSeqInit;
+ final SnippetCounter instanceLoopInit;
+ final SnippetCounter arrayLoopInit;
+ final SnippetCounter stub;
+ }
+
+ public static class Templates extends AbstractTemplates {
+
+ private final SnippetInfo allocateInstance = snippet(NewObjectSnippets.class, "allocateInstance", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
+ private final SnippetInfo allocateInstancePIC = snippet(NewObjectSnippets.class, "allocateInstancePIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
+ TLAB_END_LOCATION);
+ private final SnippetInfo allocateArray = snippet(NewObjectSnippets.class, "allocateArray", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
+ private final SnippetInfo allocateArrayPIC = snippet(NewObjectSnippets.class, "allocateArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
+ private final SnippetInfo allocatePrimitiveArrayPIC = snippet(NewObjectSnippets.class, "allocatePrimitiveArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
+ TLAB_END_LOCATION);
+ private final SnippetInfo allocateArrayDynamic = snippet(NewObjectSnippets.class, "allocateArrayDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
+ TLAB_END_LOCATION);
+ private final SnippetInfo allocateInstanceDynamic = snippet(NewObjectSnippets.class, "allocateInstanceDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
+ TLAB_END_LOCATION);
+ private final SnippetInfo newmultiarray = snippet(NewObjectSnippets.class, "newmultiarray", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
+ private final SnippetInfo newmultiarrayPIC = snippet(NewObjectSnippets.class, "newmultiarrayPIC", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
+ private final SnippetInfo verifyHeap = snippet(NewObjectSnippets.class, "verifyHeap");
+ private final GraalHotSpotVMConfig config;
+ private final Counters counters;
+
+ public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target,
+ GraalHotSpotVMConfig config) {
+ super(options, factories, providers, providers.getSnippetReflection(), target);
+ this.config = config;
+ counters = new Counters(factory);
+ }
+
+ /**
+ * Lowers a {@link NewInstanceNode}.
+ */
+ public void lower(NewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
+ StructuredGraph graph = newInstanceNode.graph();
+ HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newInstanceNode.instanceClass();
+ assert !type.isArray();
+ ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
+ int size = instanceSize(type);
+
+ OptionValues localOptions = graph.getOptions();
+ SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? allocateInstancePIC : allocateInstance;
+ Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
+ args.addConst("size", size);
+ args.add("hub", hub);
+ args.add("prototypeMarkWord", type.prototypeMarkWord());
+ args.addConst("fillContents", newInstanceNode.fillContents());
+ args.addConst("threadRegister", registers.getThreadRegister());
+ args.addConst("constantSize", true);
+ args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? type.toJavaName(false) : "");
+ args.addConst("options", localOptions);
+ args.addConst("counters", counters);
+
+ SnippetTemplate template = template(graph.getDebug(), args);
+ graph.getDebug().log("Lowering allocateInstance in %s: node=%s, template=%s, arguments=%s", graph, newInstanceNode, template, args);
+ template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
+ }
+
+ /**
+ * Lowers a {@link NewArrayNode}.
+ */
+ public void lower(NewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
+ StructuredGraph graph = newArrayNode.graph();
+ ResolvedJavaType elementType = newArrayNode.elementType();
+ HotSpotResolvedObjectType arrayType = (HotSpotResolvedObjectType) elementType.getArrayClass();
+ JavaKind elementKind = elementType.getJavaKind();
+ ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayType.klass(), providers.getMetaAccess(), graph);
+ final int headerSize = getArrayBaseOffset(elementKind);
+ int log2ElementSize = CodeUtil.log2(HotSpotJVMCIRuntimeProvider.getArrayIndexScale(elementKind));
+
+ OptionValues localOptions = graph.getOptions();
+ SnippetInfo snippet;
+ if (GeneratePIC.getValue(localOptions)) {
+ if (elementType.isPrimitive()) {
+ snippet = allocatePrimitiveArrayPIC;
+ } else {
+ snippet = allocateArrayPIC;
+ }
+ } else {
+ snippet = allocateArray;
+ }
+
+ Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
+ args.add("hub", hub);
+ ValueNode length = newArrayNode.length();
+ args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
+ assert arrayType.prototypeMarkWord() == lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord() : "all array types are assumed to have the same prototypeMarkWord";
+ args.add("prototypeMarkWord", arrayType.prototypeMarkWord());
+ args.addConst("headerSize", headerSize);
+ args.addConst("log2ElementSize", log2ElementSize);
+ args.addConst("fillContents", newArrayNode.fillContents());
+ args.addConst("threadRegister", registers.getThreadRegister());
+ args.addConst("maybeUnroll", length.isConstant());
+ args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? arrayType.toJavaName(false) : "");
+ args.addConst("options", localOptions);
+ args.addConst("counters", counters);
+ SnippetTemplate template = template(graph.getDebug(), args);
+ graph.getDebug().log("Lowering allocateArray in %s: node=%s, template=%s, arguments=%s", graph, newArrayNode, template, args);
+ template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
+ }
+
+ public void lower(DynamicNewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
+ Arguments args = new Arguments(allocateInstanceDynamic, newInstanceNode.graph().getGuardsStage(), tool.getLoweringStage());
+ OptionValues localOptions = newInstanceNode.getOptions();
+ args.add("type", newInstanceNode.getInstanceType());
+ ValueNode classClass = newInstanceNode.getClassClass();
+ assert classClass != null;
+ args.add("classClass", classClass);
+ args.addConst("fillContents", newInstanceNode.fillContents());
+ args.addConst("threadRegister", registers.getThreadRegister());
+ args.addConst("options", localOptions);
+ args.addConst("counters", counters);
+
+ SnippetTemplate template = template(newInstanceNode.getDebug(), args);
+ template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
+ }
+
+ public void lower(DynamicNewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
+ StructuredGraph graph = newArrayNode.graph();
+ OptionValues localOptions = graph.getOptions();
+ Arguments args = new Arguments(allocateArrayDynamic, newArrayNode.graph().getGuardsStage(), tool.getLoweringStage());
+ args.add("elementType", newArrayNode.getElementType());
+ ValueNode voidClass = newArrayNode.getVoidClass();
+ assert voidClass != null;
+ args.add("voidClass", voidClass);
+ ValueNode length = newArrayNode.length();
+ args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
+ args.addConst("fillContents", newArrayNode.fillContents());
+ args.addConst("threadRegister", registers.getThreadRegister());
+ /*
+ * We use Kind.Illegal as a marker value instead of null because constant snippet
+ * parameters cannot be null.
+ */
+ args.addConst("knownElementKind", newArrayNode.getKnownElementKind() == null ? JavaKind.Illegal : newArrayNode.getKnownElementKind());
+ if (newArrayNode.getKnownElementKind() != null) {
+ args.addConst("knownLayoutHelper", lookupArrayClass(tool, newArrayNode.getKnownElementKind()).layoutHelper());
+ } else {
+ args.addConst("knownLayoutHelper", 0);
+ }
+ args.add("prototypeMarkWord", lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord());
+ args.addConst("options", localOptions);
+ args.addConst("counters", counters);
+ SnippetTemplate template = template(graph.getDebug(), args);
+ template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
+ }
+
+ private static HotSpotResolvedObjectType lookupArrayClass(LoweringTool tool, JavaKind kind) {
+ return (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(kind == JavaKind.Object ? Object.class : kind.toJavaClass()).getArrayClass();
+ }
+
+ public void lower(NewMultiArrayNode newmultiarrayNode, LoweringTool tool) {
+ StructuredGraph graph = newmultiarrayNode.graph();
+ OptionValues localOptions = graph.getOptions();
+ int rank = newmultiarrayNode.dimensionCount();
+ ValueNode[] dims = new ValueNode[rank];
+ for (int i = 0; i < newmultiarrayNode.dimensionCount(); i++) {
+ dims[i] = newmultiarrayNode.dimension(i);
+ }
+ HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newmultiarrayNode.type();
+ ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
+
+ SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? newmultiarrayPIC : newmultiarray;
+ Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
+ args.add("hub", hub);
+ args.addConst("rank", rank);
+ args.addVarargs("dimensions", int.class, StampFactory.forKind(JavaKind.Int), dims);
+ template(newmultiarrayNode.getDebug(), args).instantiate(providers.getMetaAccess(), newmultiarrayNode, DEFAULT_REPLACER, args);
+ }
+
+ private static int instanceSize(HotSpotResolvedObjectType type) {
+ int size = type.instanceSize();
+ assert size >= 0;
+ return size;
+ }
+
+ public void lower(VerifyHeapNode verifyHeapNode, HotSpotRegistersProvider registers, LoweringTool tool) {
+ if (config.cAssertions) {
+ Arguments args = new Arguments(verifyHeap, verifyHeapNode.graph().getGuardsStage(), tool.getLoweringStage());
+ args.addConst("threadRegister", registers.getThreadRegister());
+
+ SnippetTemplate template = template(verifyHeapNode.getDebug(), args);
+ template.instantiate(providers.getMetaAccess(), verifyHeapNode, DEFAULT_REPLACER, args);
+ } else {
+ GraphUtil.removeFixedWithUnusedInputs(verifyHeapNode);
+ }
+ }
+ }
+}