diff -r 0152ad7b38b8 -r 6df94ce3ab2f src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/NewObjectSnippets.java --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/NewObjectSnippets.java Tue Sep 24 08:54:08 2019 -0700 +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/NewObjectSnippets.java Tue Sep 24 12:47:15 2019 -0400 @@ -24,6 +24,7 @@ package org.graalvm.compiler.hotspot.replacements; +import static jdk.vm.ci.meta.DeoptimizationAction.InvalidateRecompile; import static jdk.vm.ci.meta.DeoptimizationAction.None; import static jdk.vm.ci.meta.DeoptimizationReason.RuntimeConstraint; import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC; @@ -38,7 +39,7 @@ import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY; import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY_OR_NULL; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION; -import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_STATE_LOCATION; +import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_INIT_STATE_LOCATION; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION; @@ -54,6 +55,7 @@ import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize; +import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceKlassStateBeingInitialized; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift; @@ -61,6 +63,8 @@ import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset; +import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readInstanceKlassInitState; +import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readInstanceKlassInitThread; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop; @@ -73,9 +77,11 @@ import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocationsContext; import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCastToSnippetReplaceeStamp; import static org.graalvm.compiler.nodes.PiNode.piCastToSnippetReplaceeStamp; +import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.DEOPT_PROBABILITY; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY; +import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.VERY_FAST_PATH_PROBABILITY; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; import static org.graalvm.compiler.replacements.ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED; import static org.graalvm.compiler.replacements.ReplacementsUtil.runtimeAssert; @@ -99,6 +105,7 @@ import org.graalvm.compiler.hotspot.meta.HotSpotProviders; import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider; import org.graalvm.compiler.hotspot.nodes.DimensionsNode; +import org.graalvm.compiler.hotspot.nodes.KlassBeingInitializedCheckNode; import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyFixedNode; import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyNode; import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp; @@ -112,6 +119,7 @@ import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.debug.DynamicCounterNode; import org.graalvm.compiler.nodes.debug.VerifyHeapNode; +import org.graalvm.compiler.nodes.extended.BranchProbabilityNode; import org.graalvm.compiler.nodes.extended.ForeignCallNode; import org.graalvm.compiler.nodes.extended.MembarNode; import org.graalvm.compiler.nodes.java.DynamicNewArrayNode; @@ -290,12 +298,12 @@ @ConstantParameter boolean emitMemoryBarrier, @ConstantParameter Register threadRegister, @ConstantParameter Counters counters) { - if (probability(SLOW_PATH_PROBABILITY, type == null)) { + if (probability(DEOPT_PROBABILITY, type == null)) { DeoptimizeNode.deopt(None, RuntimeConstraint); } Class nonNullType = PiNode.piCastNonNullClass(type, SnippetAnchorNode.anchor()); - if (probability(SLOW_PATH_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) { + if (probability(DEOPT_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) { DeoptimizeNode.deopt(None, RuntimeConstraint); } @@ -312,7 +320,7 @@ if (probability(FAST_PATH_PROBABILITY, !hub.isNull())) { KlassPointer nonNullHub = ClassGetHubNode.piCastNonNull(hub, SnippetAnchorNode.anchor()); - if (probability(FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(nonNullHub))) { + if (probability(VERY_FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(nonNullHub))) { int layoutHelper = readLayoutHelper(nonNullHub); /* * src/share/vm/oops/klass.hpp: For instances, layout helper is a positive number, @@ -351,12 +359,12 @@ @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext, - @ConstantParameter boolean useBulkZeroing, + @ConstantParameter int bulkZeroingStride, @ConstantParameter Counters counters) { // Primitive array types are eagerly pre-resolved. We can use a floating load. KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub); return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, - emitMemoryBarrier, threadRegister, maybeUnroll, typeContext, useBulkZeroing, counters); + emitMemoryBarrier, threadRegister, maybeUnroll, typeContext, bulkZeroingStride, counters); } @Snippet @@ -370,12 +378,12 @@ @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext, - @ConstantParameter boolean useBulkZeroing, + @ConstantParameter int bulkZeroingStride, @ConstantParameter Counters counters) { // Array type would be resolved by dominating resolution. KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub); return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, - emitMemoryBarrier, threadRegister, maybeUnroll, typeContext, useBulkZeroing, counters); + emitMemoryBarrier, threadRegister, maybeUnroll, typeContext, bulkZeroingStride, counters); } @Snippet @@ -389,7 +397,7 @@ @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext, - @ConstantParameter boolean useBulkZeroing, + @ConstantParameter int bulkZeroingStride, @ConstantParameter Counters counters) { Object result = allocateArrayImpl(hub, length, @@ -400,7 +408,7 @@ emitMemoryBarrier, threadRegister, maybeUnroll, typeContext, - useBulkZeroing, + bulkZeroingStride, counters); return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length); } @@ -424,7 +432,7 @@ Register threadRegister, boolean maybeUnroll, String typeContext, - boolean useBulkZeroing, + int bulkZeroingStride, Counters counters) { Object result; long allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize); @@ -440,7 +448,7 @@ if (theCounters != null && theCounters.arrayLoopInit != null) { theCounters.arrayLoopInit.inc(); } - result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, emitMemoryBarrier, maybeUnroll, useBulkZeroing, counters); + result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, emitMemoryBarrier, maybeUnroll, bulkZeroingStride, counters); } else { result = newArrayStub(hub, length); } @@ -484,7 +492,7 @@ * Deoptimizes if {@code obj == null} otherwise returns {@code obj}. */ private static Object nonNullOrDeopt(Object obj) { - if (obj == null) { + if (BranchProbabilityNode.probability(BranchProbabilityNode.DEOPT_PROBABILITY, obj == null)) { DeoptimizeNode.deopt(None, RuntimeConstraint); } return obj; @@ -505,11 +513,11 @@ @ConstantParameter Register threadRegister, @ConstantParameter JavaKind knownElementKind, @ConstantParameter int knownLayoutHelper, - @ConstantParameter boolean useBulkZeroing, + @ConstantParameter int bulkZeroingStride, Word prototypeMarkWord, @ConstantParameter Counters counters) { Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, emitMemoryBarrier, threadRegister, knownElementKind, - knownLayoutHelper, useBulkZeroing, prototypeMarkWord, counters); + knownLayoutHelper, bulkZeroingStride, prototypeMarkWord, counters); return result; } @@ -521,7 +529,7 @@ Register threadRegister, JavaKind knownElementKind, int knownLayoutHelper, - boolean useBulkZeroing, + int bulkZeroingStride, Word prototypeMarkWord, Counters counters) { /* @@ -534,12 +542,12 @@ } KlassPointer klass = loadKlassFromObject(elementType, arrayKlassOffset(INJECTED_VMCONFIG), CLASS_ARRAY_KLASS_LOCATION); - if (klass.isNull()) { + if (probability(DEOPT_PROBABILITY, klass.isNull())) { DeoptimizeNode.deopt(None, RuntimeConstraint); } KlassPointer nonNullKlass = ClassGetHubNode.piCastNonNull(klass, SnippetAnchorNode.anchor()); - if (length < 0) { + if (probability(DEOPT_PROBABILITY, length < 0)) { DeoptimizeNode.deopt(None, RuntimeConstraint); } int layoutHelper; @@ -566,7 +574,7 @@ int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG); Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, - emitMemoryBarrier, threadRegister, false, "dynamic type", useBulkZeroing, counters); + emitMemoryBarrier, threadRegister, false, "dynamic type", bulkZeroingStride, counters); return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length); } @@ -614,21 +622,21 @@ * Zero uninitialized memory in a newly allocated object, unrolling as necessary and ensuring * that stores are aligned. * - * @param size number of bytes to zero * @param memory beginning of object which is being zeroed - * @param constantSize is {@code size} known to be constant in the snippet - * @param startOffset offset to begin zeroing. May not be word aligned. + * @param startOffset offset to begin zeroing (inclusive). May not be word aligned. + * @param endOffset offset to stop zeroing (exclusive). May not be word aligned. + * @param isEndOffsetConstant is {@code endOffset} known to be constant in the snippet * @param manualUnroll maximally unroll zeroing - * @param useBulkZeroing apply bulk zeroing + * @param bulkZeroingStride stride of bulk zeroing supported by the backend */ - private static void zeroMemory(long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, - boolean useBulkZeroing, Counters counters) { - fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, useBulkZeroing, counters); + private static void zeroMemory(Word memory, int startOffset, long endOffset, boolean isEndOffsetConstant, boolean manualUnroll, + int bulkZeroingStride, Counters counters) { + fillMemory(0, memory, startOffset, endOffset, isEndOffsetConstant, manualUnroll, bulkZeroingStride, counters); } - private static void fillMemory(long value, long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, - boolean useBulkZeroing, Counters counters) { - ReplacementsUtil.runtimeAssert((size & 0x7) == 0, "unaligned object size"); + private static void fillMemory(long value, Word memory, int startOffset, long offsetLimit, boolean constantOffsetLimit, boolean manualUnroll, + int bulkZeroingStride, Counters counters) { + ReplacementsUtil.runtimeAssert((offsetLimit & 0x7) == 0, "unaligned object size"); int offset = startOffset; if ((offset & 0x7) != 0) { memory.writeInt(offset, (int) value, LocationIdentity.init()); @@ -636,8 +644,8 @@ } ReplacementsUtil.runtimeAssert((offset & 0x7) == 0, "unaligned offset"); Counters theCounters = counters; - if (manualUnroll && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) { - ReplacementsUtil.staticAssert(!constantSize, "size shouldn't be constant at instantiation time"); + if (manualUnroll && ((offsetLimit - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) { + ReplacementsUtil.staticAssert(!constantOffsetLimit, "size shouldn't be constant at instantiation time"); // This case handles arrays of constant length. Instead of having a snippet variant for // each length, generate a chain of stores of maximum length. Once it's inlined the // break statement will trim excess stores. @@ -647,7 +655,7 @@ explodeLoop(); for (int i = 0; i < MAX_UNROLLED_OBJECT_ZEROING_STORES; i++, offset += 8) { - if (offset == size) { + if (offset == offsetLimit) { break; } memory.initializeLong(offset, value, LocationIdentity.init()); @@ -655,13 +663,13 @@ } else { // Use Word instead of int to avoid extension to long in generated code Word off = WordFactory.signed(offset); - if (useBulkZeroing && value == 0 && probability(SLOW_PATH_PROBABILITY, (size - offset) >= getMinimalBulkZeroingSize(INJECTED_OPTIONVALUES))) { + if (bulkZeroingStride > 0 && value == 0 && probability(SLOW_PATH_PROBABILITY, (offsetLimit - offset) >= getMinimalBulkZeroingSize(INJECTED_OPTIONVALUES))) { if (theCounters != null && theCounters.instanceBulkInit != null) { theCounters.instanceBulkInit.inc(); } - ZeroMemoryNode.zero(memory.add(off), size - offset, LocationIdentity.init()); + ZeroMemoryNode.zero(memory.add(off), offsetLimit - offset, LocationIdentity.init()); } else { - if (constantSize && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) { + if (constantOffsetLimit && ((offsetLimit - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) { if (theCounters != null && theCounters.instanceSeqInit != null) { theCounters.instanceSeqInit.inc(); } @@ -671,7 +679,7 @@ theCounters.instanceLoopInit.inc(); } } - for (; off.rawValue() < size; off = off.add(8)) { + for (; off.rawValue() < offsetLimit; off = off.add(8)) { memory.initializeLong(off, value, LocationIdentity.init()); } } @@ -687,14 +695,15 @@ * Fill uninitialized memory with garbage value in a newly allocated object, unrolling as * necessary and ensuring that stores are aligned. * - * @param size number of bytes to zero * @param memory beginning of object which is being zeroed - * @param constantSize is {@code size} known to be constant in the snippet - * @param startOffset offset to begin zeroing. May not be word aligned. + * @param startOffset offset to begin filling garbage value (inclusive). May not be word + * aligned. + * @param endOffset offset to stop filling garbage value (exclusive). May not be word aligned. + * @param isEndOffsetConstant is {@code endOffset} known to be constant in the snippet * @param manualUnroll maximally unroll zeroing */ - private static void fillWithGarbage(long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) { - fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, false, counters); + private static void fillWithGarbage(Word memory, int startOffset, long endOffset, boolean isEndOffsetConstant, boolean manualUnroll, Counters counters) { + fillMemory(0xfefefefefefefefeL, memory, startOffset, endOffset, isEndOffsetConstant, manualUnroll, 0, counters); } /** @@ -711,9 +720,9 @@ Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord; initializeObjectHeader(memory, prototypeMarkWord, hub); if (fillContents) { - zeroMemory(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, false, counters); + zeroMemory(memory, instanceHeaderSize(INJECTED_VMCONFIG), size, constantSize, false, 0, counters); } else if (REPLACEMENTS_ASSERTIONS_ENABLED) { - fillWithGarbage(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters); + fillWithGarbage(memory, instanceHeaderSize(INJECTED_VMCONFIG), size, constantSize, false, counters); } if (emitMemoryBarrier) { MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init()); @@ -733,6 +742,19 @@ } } + @Snippet + private static void threadBeingInitializedCheck(@ConstantParameter Register threadRegister, KlassPointer klass) { + int state = readInstanceKlassInitState(klass); + if (state != instanceKlassStateBeingInitialized(INJECTED_VMCONFIG)) { + // The klass is no longer being initialized so force recompilation + DeoptimizeNode.deopt(InvalidateRecompile, RuntimeConstraint); + } else if (registerAsWord(threadRegister) != readInstanceKlassInitThread(klass)) { + // The klass is being initialized but this isn't the initializing thread so + // so deopt and allow execution to resume in the interpreter where it should block. + DeoptimizeNode.deopt(None, RuntimeConstraint); + } + } + /** * Formats some allocated memory with an object header and zeroes out the rest. */ @@ -745,7 +767,7 @@ boolean fillContents, boolean emitMemoryBarrier, boolean maybeUnroll, - boolean useBulkZeroing, + int bulkZeroingStride, Counters counters) { memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, LocationIdentity.init()); /* @@ -754,9 +776,9 @@ */ initializeObjectHeader(memory, prototypeMarkWord, hub); if (fillContents) { - zeroMemory(allocationSize, memory, false, headerSize, maybeUnroll, useBulkZeroing, counters); + zeroMemory(memory, headerSize, allocationSize, false, maybeUnroll, bulkZeroingStride, counters); } else if (REPLACEMENTS_ASSERTIONS_ENABLED) { - fillWithGarbage(allocationSize, memory, false, headerSize, maybeUnroll, counters); + fillWithGarbage(memory, headerSize, allocationSize, false, maybeUnroll, counters); } if (emitMemoryBarrier) { MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init()); @@ -795,10 +817,11 @@ private final SnippetInfo allocateArrayDynamic = snippet(NewObjectSnippets.class, "allocateArrayDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION); private final SnippetInfo allocateInstanceDynamic = snippet(NewObjectSnippets.class, "allocateInstanceDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, - TLAB_END_LOCATION, PROTOTYPE_MARK_WORD_LOCATION, CLASS_STATE_LOCATION); + TLAB_END_LOCATION, PROTOTYPE_MARK_WORD_LOCATION, CLASS_INIT_STATE_LOCATION); private final SnippetInfo newmultiarray = snippet(NewObjectSnippets.class, "newmultiarray", TLAB_TOP_LOCATION, TLAB_END_LOCATION); private final SnippetInfo newmultiarrayPIC = snippet(NewObjectSnippets.class, "newmultiarrayPIC", TLAB_TOP_LOCATION, TLAB_END_LOCATION); private final SnippetInfo verifyHeap = snippet(NewObjectSnippets.class, "verifyHeap"); + private final SnippetInfo threadBeingInitializedCheck = snippet(NewObjectSnippets.class, "threadBeingInitializedCheck"); private final GraalHotSpotVMConfig config; private final Counters counters; @@ -874,7 +897,7 @@ args.addConst("threadRegister", registers.getThreadRegister()); args.addConst("maybeUnroll", length.isConstant()); args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? arrayType.toJavaName(false) : ""); - args.addConst("useBulkZeroing", tool.getLowerer().supportBulkZeroing()); + args.addConst("bulkZeroingStride", tool.getLowerer().bulkZeroingStride()); args.addConst("counters", counters); SnippetTemplate template = template(newArrayNode, args); graph.getDebug().log("Lowering allocateArray in %s: node=%s, template=%s, arguments=%s", graph, newArrayNode, template, args); @@ -918,7 +941,7 @@ } else { args.addConst("knownLayoutHelper", 0); } - args.addConst("useBulkZeroing", tool.getLowerer().supportBulkZeroing()); + args.addConst("bulkZeroingStride", tool.getLowerer().bulkZeroingStride()); args.add("prototypeMarkWord", lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord()); args.addConst("counters", counters); SnippetTemplate template = template(newArrayNode, args); @@ -965,5 +988,14 @@ GraphUtil.removeFixedWithUnusedInputs(verifyHeapNode); } } + + public void lower(KlassBeingInitializedCheckNode verifyHeapNode, HotSpotRegistersProvider registers, LoweringTool tool) { + Arguments args = new Arguments(threadBeingInitializedCheck, verifyHeapNode.graph().getGuardsStage(), tool.getLoweringStage()); + args.addConst("threadRegister", registers.getThreadRegister()); + args.add("klass", verifyHeapNode.getKlass()); + + SnippetTemplate template = template(verifyHeapNode, args); + template.instantiate(providers.getMetaAccess(), verifyHeapNode, DEFAULT_REPLACER, args); + } } }