jdk-sandbox: comparison src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/NewObjectSnippets.java

equal deleted inserted replaced

-:13588c901957
+:9cf78a70fa4f
 */
 package org.graalvm.compiler.hotspot.replacements;
+import static jdk.vm.ci.meta.DeoptimizationAction.InvalidateRecompile;
 import static jdk.vm.ci.meta.DeoptimizationAction.None;
 import static jdk.vm.ci.meta.DeoptimizationReason.RuntimeConstraint;
 import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
 import static org.graalvm.compiler.core.common.GraalOptions.MinimalBulkZeroingSize;
 import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan;
 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE;
 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE_OR_NULL;
 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY;
 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY_OR_NULL;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_INIT_STATE_LOCATION;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayAllocationSize;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceKlassStateBeingInitialized;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readInstanceKlassInitState;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readInstanceKlassInitThread;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocations;
 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocationsContext;
 import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCastToSnippetReplaceeStamp;
 import static org.graalvm.compiler.nodes.PiNode.piCastToSnippetReplaceeStamp;
+import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.DEOPT_PROBABILITY;
 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
+import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.VERY_FAST_PATH_PROBABILITY;
 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
 import static org.graalvm.compiler.replacements.ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED;
 import static org.graalvm.compiler.replacements.ReplacementsUtil.runtimeAssert;
 import static org.graalvm.compiler.replacements.ReplacementsUtil.staticAssert;
 import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
 import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
 import org.graalvm.compiler.hotspot.nodes.DimensionsNode;
+import org.graalvm.compiler.hotspot.nodes.KlassBeingInitializedCheckNode;
 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyFixedNode;
 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyNode;
 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
 import org.graalvm.compiler.hotspot.word.KlassPointer;
 import org.graalvm.compiler.nodes.ConstantNode;
 import org.graalvm.compiler.nodes.SnippetAnchorNode;
 import org.graalvm.compiler.nodes.StructuredGraph;
 import org.graalvm.compiler.nodes.ValueNode;
 import org.graalvm.compiler.nodes.debug.DynamicCounterNode;
 import org.graalvm.compiler.nodes.debug.VerifyHeapNode;
+import org.graalvm.compiler.nodes.extended.BranchProbabilityNode;
 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
 import org.graalvm.compiler.nodes.extended.MembarNode;
 import org.graalvm.compiler.nodes.java.DynamicNewArrayNode;
 import org.graalvm.compiler.nodes.java.DynamicNewInstanceNode;
 import org.graalvm.compiler.nodes.java.NewArrayNode;
 }
 }
 }
 @Snippet
-public static Object allocateInstance(@ConstantParameter long size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
+public static Object allocateInstance(@ConstantParameter long size,
-@ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext,
+KlassPointer hub,
+Word prototypeMarkWord,
+@ConstantParameter boolean fillContents,
+@ConstantParameter boolean emitMemoryBarrier,
+@ConstantParameter Register threadRegister,
+@ConstantParameter boolean constantSize,
+@ConstantParameter String typeContext,
 @ConstantParameter Counters counters) {
-return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, counters));
+return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, emitMemoryBarrier, threadRegister, constantSize, typeContext, counters));
 }
-public static Object allocateInstanceHelper(long size, KlassPointer hub, Word prototypeMarkWord, boolean fillContents,
+public static Object allocateInstanceHelper(long size,
-Register threadRegister, boolean constantSize, String typeContext, Counters counters) {
+KlassPointer hub,
+Word prototypeMarkWord,
+boolean fillContents,
+boolean emitMemoryBarrier,
+Register threadRegister,
+boolean constantSize,
+String typeContext,
+Counters counters) {
 Object result;
 Word thread = registerAsWord(threadRegister);
 Word top = readTlabTop(thread);
 Word end = readTlabEnd(thread);
 Word newTop = top.add(WordFactory.unsigned(size));
 if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
 writeTlabTop(thread, newTop);
 emitPrefetchAllocate(newTop, false);
-result = formatObject(hub, size, top, prototypeMarkWord, fillContents, constantSize, counters);
+result = formatObject(hub, size, top, prototypeMarkWord, fillContents, emitMemoryBarrier, constantSize, counters);
 } else {
 Counters theCounters = counters;
 if (theCounters != null && theCounters.stub != null) {
 theCounters.stub.inc();
 }
 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
 private static native Object newInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
 @Snippet
-public static Object allocateInstancePIC(@ConstantParameter long size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
+public static Object allocateInstancePIC(@ConstantParameter long size,
-@ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext,
+KlassPointer hub,
+Word prototypeMarkWord,
+@ConstantParameter boolean fillContents,
+@ConstantParameter boolean emitMemoryBarrier,
+@ConstantParameter Register threadRegister,
+@ConstantParameter boolean constantSize,
+@ConstantParameter String typeContext,
 @ConstantParameter Counters counters) {
 // Klass must be initialized by the time the first instance is allocated, therefore we can
 // just load it from the corresponding cell and avoid the resolution check. We have to use a
 // fixed load though, to prevent it from floating above the initialization.
 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
-return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, picHub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, counters));
+return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, picHub, prototypeMarkWord, fillContents, emitMemoryBarrier, threadRegister, constantSize, typeContext, counters));
 }
 @Snippet
-public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
+public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass,
+@ConstantParameter boolean fillContents,
+@ConstantParameter boolean emitMemoryBarrier,
+@ConstantParameter Register threadRegister,
 @ConstantParameter Counters counters) {
-if (probability(SLOW_PATH_PROBABILITY, type == null)) {
+if (probability(DEOPT_PROBABILITY, type == null)) {
 DeoptimizeNode.deopt(None, RuntimeConstraint);
 }
 Class<?> nonNullType = PiNode.piCastNonNullClass(type, SnippetAnchorNode.anchor());
-if (probability(SLOW_PATH_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) {
+if (probability(DEOPT_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) {
 DeoptimizeNode.deopt(None, RuntimeConstraint);
 }
-return PiNode.piCastToSnippetReplaceeStamp(allocateInstanceDynamicHelper(type, fillContents, threadRegister, counters, nonNullType));
+return PiNode.piCastToSnippetReplaceeStamp(allocateInstanceDynamicHelper(type, fillContents, emitMemoryBarrier, threadRegister, counters, nonNullType));
 }
-private static Object allocateInstanceDynamicHelper(Class<?> type, boolean fillContents, Register threadRegister, Counters counters, Class<?> nonNullType) {
+private static Object allocateInstanceDynamicHelper(Class<?> type,
+boolean fillContents,
+boolean emitMemoryBarrier,
+Register threadRegister,
+Counters counters,
+Class<?> nonNullType) {
 KlassPointer hub = ClassGetHubNode.readClass(nonNullType);
 if (probability(FAST_PATH_PROBABILITY, !hub.isNull())) {
 KlassPointer nonNullHub = ClassGetHubNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
-if (probability(FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(nonNullHub))) {
+if (probability(VERY_FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(nonNullHub))) {
 int layoutHelper = readLayoutHelper(nonNullHub);
 /*
 * src/share/vm/oops/klass.hpp: For instances, layout helper is a positive number,
 * the instance size. This size is already passed through align_object_size and
 * scaled to bytes. The low order bit is set if instances of this class cannot be
 Word prototypeMarkWord = nonNullHub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
 /*
 * FIXME(je,ds): we should actually pass typeContext instead of "" but late
 * binding of parameters is not yet supported by the GraphBuilderPlugin system.
 */
-return allocateInstanceHelper(layoutHelper, nonNullHub, prototypeMarkWord, fillContents, threadRegister, false, "", counters);
+return allocateInstanceHelper(layoutHelper, nonNullHub, prototypeMarkWord, fillContents, emitMemoryBarrier, threadRegister, false, "", counters);
 }
 } else {
 DeoptimizeNode.deopt(None, RuntimeConstraint);
 }
 }
 int length,
 Word prototypeMarkWord,
 @ConstantParameter int headerSize,
 @ConstantParameter int log2ElementSize,
 @ConstantParameter boolean fillContents,
+@ConstantParameter boolean emitMemoryBarrier,
 @ConstantParameter Register threadRegister,
 @ConstantParameter boolean maybeUnroll,
 @ConstantParameter String typeContext,
-@ConstantParameter boolean useBulkZeroing,
+@ConstantParameter boolean supportsBulkZeroing,
 @ConstantParameter Counters counters) {
 // Primitive array types are eagerly pre-resolved. We can use a floating load.
 KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,
-threadRegister, maybeUnroll, typeContext, useBulkZeroing, counters);
+emitMemoryBarrier, threadRegister, maybeUnroll, typeContext, supportsBulkZeroing, counters);
 }
 @Snippet
 public static Object allocateArrayPIC(KlassPointer hub,
 int length,
 Word prototypeMarkWord,
 @ConstantParameter int headerSize,
 @ConstantParameter int log2ElementSize,
 @ConstantParameter boolean fillContents,
+@ConstantParameter boolean emitMemoryBarrier,
 @ConstantParameter Register threadRegister,
 @ConstantParameter boolean maybeUnroll,
 @ConstantParameter String typeContext,
-@ConstantParameter boolean useBulkZeroing,
+@ConstantParameter boolean supportsBulkZeroing,
 @ConstantParameter Counters counters) {
 // Array type would be resolved by dominating resolution.
 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,
-threadRegister, maybeUnroll, typeContext, useBulkZeroing, counters);
+emitMemoryBarrier, threadRegister, maybeUnroll, typeContext, supportsBulkZeroing, counters);
 }
 @Snippet
 public static Object allocateArray(KlassPointer hub,
 int length,
 Word prototypeMarkWord,
 @ConstantParameter int headerSize,
 @ConstantParameter int log2ElementSize,
 @ConstantParameter boolean fillContents,
+@ConstantParameter boolean emitMemoryBarrier,
 @ConstantParameter Register threadRegister,
 @ConstantParameter boolean maybeUnroll,
 @ConstantParameter String typeContext,
-@ConstantParameter boolean useBulkZeroing,
+@ConstantParameter boolean supportsBulkZeroing,
 @ConstantParameter Counters counters) {
 Object result = allocateArrayImpl(hub,
 length,
 prototypeMarkWord,
 headerSize,
 log2ElementSize,
 fillContents,
-threadRegister,
+emitMemoryBarrier, threadRegister,
 maybeUnroll,
 typeContext,
-useBulkZeroing,
+supportsBulkZeroing,
 counters);
 return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
 }
 /**
 @Fold
 static boolean useNullAllocationStubs(@InjectedParameter GraalHotSpotVMConfig config) {
 return config.areNullAllocationStubsAvailable();
 }
-private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister,
+private static Object allocateArrayImpl(KlassPointer hub,
-boolean maybeUnroll, String typeContext, boolean useBulkZeroing, Counters counters) {
+int length,
+Word prototypeMarkWord,
+int headerSize,
+int log2ElementSize,
+boolean fillContents,
+boolean emitMemoryBarrier,
+Register threadRegister,
+boolean maybeUnroll,
+String typeContext,
+boolean supportsBulkZeroing,
+Counters counters) {
 Object result;
 long allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize);
 Word thread = registerAsWord(threadRegister);
 Word top = readTlabTop(thread);
 Word end = readTlabEnd(thread);
 emitPrefetchAllocate(newTop, true);
 Counters theCounters = counters;
 if (theCounters != null && theCounters.arrayLoopInit != null) {
 theCounters.arrayLoopInit.inc();
 }
-result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, maybeUnroll, useBulkZeroing, counters);
+result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, emitMemoryBarrier, maybeUnroll, supportsBulkZeroing, counters);
 } else {
 result = newArrayStub(hub, length);
 }
 profileAllocation("array", allocationSize, typeContext);
 return result;
 /**
 * Deoptimizes if {@code obj == null} otherwise returns {@code obj}.
 */
 private static Object nonNullOrDeopt(Object obj) {
-if (obj == null) {
+if (BranchProbabilityNode.probability(BranchProbabilityNode.DEOPT_PROBABILITY, obj == null)) {
 DeoptimizeNode.deopt(None, RuntimeConstraint);
 }
 return obj;
 }
 @Snippet
 public static Object allocateArrayDynamic(Class<?> elementType,
 Class<?> voidClass,
 int length,
 @ConstantParameter boolean fillContents,
+@ConstantParameter boolean emitMemoryBarrier,
 @ConstantParameter Register threadRegister,
 @ConstantParameter JavaKind knownElementKind,
 @ConstantParameter int knownLayoutHelper,
-@ConstantParameter boolean useBulkZeroing,
+@ConstantParameter boolean supportsBulkZeroing,
 Word prototypeMarkWord,
 @ConstantParameter Counters counters) {
-Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, threadRegister, knownElementKind,
+Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, emitMemoryBarrier, threadRegister, knownElementKind,
-knownLayoutHelper, useBulkZeroing, prototypeMarkWord, counters);
+knownLayoutHelper, supportsBulkZeroing, prototypeMarkWord, counters);
 return result;
 }
-private static Object allocateArrayDynamicImpl(Class<?> elementType, Class<?> voidClass, int length, boolean fillContents, Register threadRegister, JavaKind knownElementKind,
+private static Object allocateArrayDynamicImpl(Class<?> elementType,
-int knownLayoutHelper, boolean useBulkZeroing, Word prototypeMarkWord, Counters counters) {
+Class<?> voidClass,
+int length,
+boolean fillContents,
+boolean emitMemoryBarrier,
+Register threadRegister,
+JavaKind knownElementKind,
+int knownLayoutHelper,
+boolean supportsBulkZeroing,
+Word prototypeMarkWord,
+Counters counters) {
 /*
 * We only need the dynamic check for void when we have no static information from
 * knownElementKind.
 */
 staticAssert(knownElementKind != JavaKind.Void, "unsupported knownElementKind");
 if (knownElementKind == JavaKind.Illegal && probability(SLOW_PATH_PROBABILITY, elementType == null || DynamicNewArrayNode.throwsIllegalArgumentException(elementType, voidClass))) {
 DeoptimizeNode.deopt(None, RuntimeConstraint);
 }
 KlassPointer klass = loadKlassFromObject(elementType, arrayKlassOffset(INJECTED_VMCONFIG), CLASS_ARRAY_KLASS_LOCATION);
-if (klass.isNull()) {
+if (probability(DEOPT_PROBABILITY, klass.isNull())) {
 DeoptimizeNode.deopt(None, RuntimeConstraint);
 }
 KlassPointer nonNullKlass = ClassGetHubNode.piCastNonNull(klass, SnippetAnchorNode.anchor());
-if (length < 0) {
+if (probability(DEOPT_PROBABILITY, length < 0)) {
 DeoptimizeNode.deopt(None, RuntimeConstraint);
 }
 int layoutHelper;
 if (knownElementKind == JavaKind.Illegal) {
 layoutHelper = readLayoutHelper(nonNullKlass);
 int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
 int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
 Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,
-threadRegister, false, "dynamic type", useBulkZeroing, counters);
+emitMemoryBarrier, threadRegister, false, "dynamic type", supportsBulkZeroing, counters);
 return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
 }
 /**
 * Calls the runtime stub for implementing MULTIANEWARRAY.
 /**
 * Zero uninitialized memory in a newly allocated object, unrolling as necessary and ensuring
 * that stores are aligned.
 *
-* @param size number of bytes to zero
 * @param memory beginning of object which is being zeroed
-* @param constantSize is {@code size} known to be constant in the snippet
+* @param startOffset offset to begin zeroing (inclusive). May not be word aligned.
-* @param startOffset offset to begin zeroing. May not be word aligned.
+* @param endOffset offset to stop zeroing (exclusive). May not be word aligned.
+* @param isEndOffsetConstant is {@code endOffset} known to be constant in the snippet
 * @param manualUnroll maximally unroll zeroing
-* @param useBulkZeroing apply bulk zeroing
+* @param supportsBulkZeroing whether bulk zeroing is supported by the backend
 */
-private static void zeroMemory(long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll,
+private static void zeroMemory(Word memory, int startOffset, long endOffset, boolean isEndOffsetConstant, boolean manualUnroll,
-boolean useBulkZeroing, Counters counters) {
+boolean supportsBulkZeroing, Counters counters) {
-fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, useBulkZeroing, counters);
+fillMemory(0, memory, startOffset, endOffset, isEndOffsetConstant, manualUnroll, supportsBulkZeroing, counters);
 }
-private static void fillMemory(long value, long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll,
+private static void fillMemory(long value, Word memory, int startOffset, long endOffset, boolean constantOffsetLimit, boolean manualUnroll,
-boolean useBulkZeroing, Counters counters) {
+boolean supportsBulkZeroing, Counters counters) {
-ReplacementsUtil.runtimeAssert((size & 0x7) == 0, "unaligned object size");
+ReplacementsUtil.runtimeAssert((endOffset & 0x7) == 0, "unaligned object size");
 int offset = startOffset;
 if ((offset & 0x7) != 0) {
 memory.writeInt(offset, (int) value, LocationIdentity.init());
 offset += 4;
 }
 ReplacementsUtil.runtimeAssert((offset & 0x7) == 0, "unaligned offset");
 Counters theCounters = counters;
-if (manualUnroll && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
+if (manualUnroll && ((endOffset - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
-ReplacementsUtil.staticAssert(!constantSize, "size shouldn't be constant at instantiation time");
+ReplacementsUtil.staticAssert(!constantOffsetLimit, "size shouldn't be constant at instantiation time");
 // This case handles arrays of constant length. Instead of having a snippet variant for
 // each length, generate a chain of stores of maximum length. Once it's inlined the
 // break statement will trim excess stores.
 if (theCounters != null && theCounters.instanceSeqInit != null) {
 theCounters.instanceSeqInit.inc();
 }
 explodeLoop();
 for (int i = 0; i < MAX_UNROLLED_OBJECT_ZEROING_STORES; i++, offset += 8) {
-if (offset == size) {
+if (offset == endOffset) {
 break;
 }
 memory.initializeLong(offset, value, LocationIdentity.init());
 }
 } else {
 // Use Word instead of int to avoid extension to long in generated code
 Word off = WordFactory.signed(offset);
-if (useBulkZeroing && probability(SLOW_PATH_PROBABILITY, size >= getMinimalBulkZeroingSize(INJECTED_OPTIONVALUES))) {
+if (supportsBulkZeroing && value == 0 && probability(SLOW_PATH_PROBABILITY, (endOffset - offset) >= getMinimalBulkZeroingSize(INJECTED_OPTIONVALUES))) {
 if (theCounters != null && theCounters.instanceBulkInit != null) {
 theCounters.instanceBulkInit.inc();
 }
-ZeroMemoryNode.zero(memory.add(off), size - offset, LocationIdentity.init());
+ZeroMemoryNode.zero(memory.add(off), endOffset - offset, true, LocationIdentity.init());
 } else {
-if (constantSize && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
+if (constantOffsetLimit && ((endOffset - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
 if (theCounters != null && theCounters.instanceSeqInit != null) {
 theCounters.instanceSeqInit.inc();
 }
 explodeLoop();
 } else {
 if (theCounters != null && theCounters.instanceLoopInit != null) {
 theCounters.instanceLoopInit.inc();
 }
 }
-for (; off.rawValue() < size; off = off.add(8)) {
+for (; off.rawValue() < endOffset; off = off.add(8)) {
 memory.initializeLong(off, value, LocationIdentity.init());
 }
 }
 }
 }
 @Fold
 /**
 * Fill uninitialized memory with garbage value in a newly allocated object, unrolling as
 * necessary and ensuring that stores are aligned.
 *
-* @param size number of bytes to zero
 * @param memory beginning of object which is being zeroed
-* @param constantSize is {@code  size} known to be constant in the snippet
+* @param startOffset offset to begin filling garbage value (inclusive). May not be word
-* @param startOffset offset to begin zeroing. May not be word aligned.
+*            aligned.
+* @param endOffset offset to stop filling garbage value (exclusive). May not be word aligned.
+* @param isEndOffsetConstant is {@code  endOffset} known to be constant in the snippet
 * @param manualUnroll maximally unroll zeroing
 */
-private static void fillWithGarbage(long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
+private static void fillWithGarbage(Word memory, int startOffset, long endOffset, boolean isEndOffsetConstant, boolean manualUnroll, Counters counters) {
-fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, false, counters);
+fillMemory(0xfefefefefefefefeL, memory, startOffset, endOffset, isEndOffsetConstant, manualUnroll, false, counters);
 }
 /**
 * Formats some allocated memory with an object header and zeroes out the rest.
 */
-private static Object formatObject(KlassPointer hub, long size, Word memory, Word compileTimePrototypeMarkWord, boolean fillContents, boolean constantSize, Counters counters) {
+private static Object formatObject(KlassPointer hub,
+long size,
+Word memory,
+Word compileTimePrototypeMarkWord,
+boolean fillContents,
+boolean emitMemoryBarrier,
+boolean constantSize,
+Counters counters) {
 Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord;
 initializeObjectHeader(memory, prototypeMarkWord, hub);
 if (fillContents) {
-zeroMemory(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, false, counters);
+zeroMemory(memory, instanceHeaderSize(INJECTED_VMCONFIG), size, constantSize, false, false, counters);
 } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
-fillWithGarbage(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
+fillWithGarbage(memory, instanceHeaderSize(INJECTED_VMCONFIG), size, constantSize, false, counters);
 }
-MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
+if (emitMemoryBarrier) {
+MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
+}
 return memory.toObjectNonNull();
 }
 @Snippet
 private static void verifyHeap(@ConstantParameter Register threadRegister) {
 AssertionSnippets.vmMessageC(AssertionSnippets.ASSERTION_VM_MESSAGE_C, true, cstring("overzeroing of TLAB detected"), 0L, 0L, 0L);
 }
 }
 }
+@Snippet
+private static void threadBeingInitializedCheck(@ConstantParameter Register threadRegister, KlassPointer klass) {
+int state = readInstanceKlassInitState(klass);
+if (state != instanceKlassStateBeingInitialized(INJECTED_VMCONFIG)) {
+// The klass is no longer being initialized so force recompilation
+DeoptimizeNode.deopt(InvalidateRecompile, RuntimeConstraint);
+} else if (registerAsWord(threadRegister) != readInstanceKlassInitThread(klass)) {
+// The klass is being initialized but this isn't the initializing thread so
+// so deopt and allow execution to resume in the interpreter where it should block.
+DeoptimizeNode.deopt(None, RuntimeConstraint);
+}
+}
 /**
 * Formats some allocated memory with an object header and zeroes out the rest.
 */
-private static Object formatArray(KlassPointer hub, long allocationSize, int length, int headerSize, Word memory, Word prototypeMarkWord, boolean fillContents, boolean maybeUnroll,
+private static Object formatArray(KlassPointer hub,
-boolean useBulkZeroing, Counters counters) {
+long allocationSize,
+int length,
+int headerSize,
+Word memory,
+Word prototypeMarkWord,
+boolean fillContents,
+boolean emitMemoryBarrier,
+boolean maybeUnroll,
+boolean supportsBulkZeroing,
+Counters counters) {
 memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, LocationIdentity.init());
 /*
 * store hub last as the concurrent garbage collectors assume length is valid if hub field
 * is not null
 */
 initializeObjectHeader(memory, prototypeMarkWord, hub);
 if (fillContents) {
-zeroMemory(allocationSize, memory, false, headerSize, maybeUnroll, useBulkZeroing, counters);
+zeroMemory(memory, headerSize, allocationSize, false, maybeUnroll, supportsBulkZeroing, counters);
 } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
-fillWithGarbage(allocationSize, memory, false, headerSize, maybeUnroll, counters);
+fillWithGarbage(memory, headerSize, allocationSize, false, maybeUnroll, counters);
 }
-MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
+if (emitMemoryBarrier) {
+MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
+}
 return memory.toObjectNonNull();
 }
 static class Counters {
 Counters(SnippetCounter.Group.Factory factory) {
 final SnippetCounter stub;
 }
 public static class Templates extends AbstractTemplates {
-private final SnippetInfo allocateInstance = snippet(NewObjectSnippets.class, "allocateInstance", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
+private final SnippetInfo allocateInstance = snippet(NewObjectSnippets.class, "allocateInstance", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION,
+PROTOTYPE_MARK_WORD_LOCATION);
 private final SnippetInfo allocateInstancePIC = snippet(NewObjectSnippets.class, "allocateInstancePIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
-TLAB_END_LOCATION);
+TLAB_END_LOCATION, PROTOTYPE_MARK_WORD_LOCATION);
 private final SnippetInfo allocateArray = snippet(NewObjectSnippets.class, "allocateArray", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
 private final SnippetInfo allocateArrayPIC = snippet(NewObjectSnippets.class, "allocateArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
 private final SnippetInfo allocatePrimitiveArrayPIC = snippet(NewObjectSnippets.class, "allocatePrimitiveArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
 TLAB_END_LOCATION);
 private final SnippetInfo allocateArrayDynamic = snippet(NewObjectSnippets.class, "allocateArrayDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
 TLAB_END_LOCATION);
 private final SnippetInfo allocateInstanceDynamic = snippet(NewObjectSnippets.class, "allocateInstanceDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
-TLAB_END_LOCATION);
+TLAB_END_LOCATION, PROTOTYPE_MARK_WORD_LOCATION, CLASS_INIT_STATE_LOCATION);
 private final SnippetInfo newmultiarray = snippet(NewObjectSnippets.class, "newmultiarray", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
 private final SnippetInfo newmultiarrayPIC = snippet(NewObjectSnippets.class, "newmultiarrayPIC", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
 private final SnippetInfo verifyHeap = snippet(NewObjectSnippets.class, "verifyHeap");
+private final SnippetInfo threadBeingInitializedCheck = snippet(NewObjectSnippets.class, "threadBeingInitializedCheck");
 private final GraalHotSpotVMConfig config;
 private final Counters counters;
 public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target,
 GraalHotSpotVMConfig config) {
 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
 args.addConst("size", size);
 args.add("hub", hub);
 args.add("prototypeMarkWord", type.prototypeMarkWord());
 args.addConst("fillContents", newInstanceNode.fillContents());
+args.addConst("emitMemoryBarrier", newInstanceNode.emitMemoryBarrier());
 args.addConst("threadRegister", registers.getThreadRegister());
 args.addConst("constantSize", true);
 args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? type.toJavaName(false) : "");
 args.addConst("counters", counters);
 assert arrayType.prototypeMarkWord() == lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord() : "all array types are assumed to have the same prototypeMarkWord";
 args.add("prototypeMarkWord", arrayType.prototypeMarkWord());
 args.addConst("headerSize", headerSize);
 args.addConst("log2ElementSize", log2ElementSize);
 args.addConst("fillContents", newArrayNode.fillContents());
+args.addConst("emitMemoryBarrier", newArrayNode.emitMemoryBarrier());
 args.addConst("threadRegister", registers.getThreadRegister());
 args.addConst("maybeUnroll", length.isConstant());
 args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? arrayType.toJavaName(false) : "");
-args.addConst("useBulkZeroing", tool.getLowerer().supportBulkZeroing());
+args.addConst("supportsBulkZeroing", tool.getLowerer().supportsBulkZeroing());
 args.addConst("counters", counters);
 SnippetTemplate template = template(newArrayNode, args);
 graph.getDebug().log("Lowering allocateArray in %s: node=%s, template=%s, arguments=%s", graph, newArrayNode, template, args);
 template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
 }
 args.add("type", newInstanceNode.getInstanceType());
 ValueNode classClass = newInstanceNode.getClassClass();
 assert classClass != null;
 args.add("classClass", classClass);
 args.addConst("fillContents", newInstanceNode.fillContents());
+args.addConst("emitMemoryBarrier", newInstanceNode.emitMemoryBarrier());
 args.addConst("threadRegister", registers.getThreadRegister());
 args.addConst("counters", counters);
 SnippetTemplate template = template(newInstanceNode, args);
 template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
 assert voidClass != null;
 args.add("voidClass", voidClass);
 ValueNode length = newArrayNode.length();
 args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
 args.addConst("fillContents", newArrayNode.fillContents());
+args.addConst("emitMemoryBarrier", newArrayNode.emitMemoryBarrier());
 args.addConst("threadRegister", registers.getThreadRegister());
 /*
 * We use Kind.Illegal as a marker value instead of null because constant snippet
 * parameters cannot be null.
 */
 if (newArrayNode.getKnownElementKind() != null) {
 args.addConst("knownLayoutHelper", lookupArrayClass(tool, newArrayNode.getKnownElementKind()).layoutHelper());
 } else {
 args.addConst("knownLayoutHelper", 0);
 }
-args.addConst("useBulkZeroing", tool.getLowerer().supportBulkZeroing());
+args.addConst("supportsBulkZeroing", tool.getLowerer().supportsBulkZeroing());
 args.add("prototypeMarkWord", lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord());
 args.addConst("counters", counters);
 SnippetTemplate template = template(newArrayNode, args);
 template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
 }
 template.instantiate(providers.getMetaAccess(), verifyHeapNode, DEFAULT_REPLACER, args);
 } else {
 GraphUtil.removeFixedWithUnusedInputs(verifyHeapNode);
 }
 }
+public void lower(KlassBeingInitializedCheckNode verifyHeapNode, HotSpotRegistersProvider registers, LoweringTool tool) {
+Arguments args = new Arguments(threadBeingInitializedCheck, verifyHeapNode.graph().getGuardsStage(), tool.getLoweringStage());
+args.addConst("threadRegister", registers.getThreadRegister());
+args.add("klass", verifyHeapNode.getKlass());
+SnippetTemplate template = template(verifyHeapNode, args);
+template.instantiate(providers.getMetaAccess(), verifyHeapNode, DEFAULT_REPLACER, args);
+}
 }
 }

branch	datagramsocketimpl-branch
changeset 58678	9cf78a70fa4f
parent 54724	62f373a53296
child 58679	9c3209ff7550