/*

 * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 */

package org.graalvm.compiler.hotspot.replacements;

import static jdk.vm.ci.meta.DeoptimizationAction.InvalidateRecompile;

import static jdk.vm.ci.meta.DeoptimizationAction.None;

import static jdk.vm.ci.meta.DeoptimizationReason.RuntimeConstraint;

import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;

import static org.graalvm.compiler.core.common.GraalOptions.MinimalBulkZeroingSize;

import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan;

import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_OPTIONVALUES;

import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_VMCONFIG;

import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_ARRAY;

import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_ARRAY_OR_NULL;

import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE;

import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE_OR_NULL;

import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY;

import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY_OR_NULL;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_INIT_STATE_LOCATION;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocateInstancePrefetchLines;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchDistance;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchLines;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchStepSize;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchStyle;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayAllocationSize;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceKlassStateBeingInitialized;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readInstanceKlassInitState;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readInstanceKlassInitThread;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;

import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;

import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocations;

import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocationsContext;

import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCastToSnippetReplaceeStamp;

import static org.graalvm.compiler.nodes.PiNode.piCastToSnippetReplaceeStamp;

import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.DEOPT_PROBABILITY;

import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;

import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;

import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;

import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.VERY_FAST_PATH_PROBABILITY;

import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;

import static org.graalvm.compiler.replacements.ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED;

import static org.graalvm.compiler.replacements.ReplacementsUtil.runtimeAssert;

import static org.graalvm.compiler.replacements.ReplacementsUtil.staticAssert;

import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;

import static org.graalvm.compiler.replacements.nodes.CStringConstant.cstring;

import static org.graalvm.compiler.replacements.nodes.ExplodeLoopNode.explodeLoop;

import org.graalvm.compiler.api.replacements.Fold;

import org.graalvm.compiler.api.replacements.Fold.InjectedParameter;

import org.graalvm.compiler.api.replacements.Snippet;

import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;

import org.graalvm.compiler.api.replacements.Snippet.VarargsParameter;

import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;

import org.graalvm.compiler.core.common.type.StampFactory;

import org.graalvm.compiler.debug.DebugHandlersFactory;

import org.graalvm.compiler.debug.GraalError;

import org.graalvm.compiler.graph.Node.ConstantNodeParameter;

import org.graalvm.compiler.graph.Node.NodeIntrinsic;

import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;

import org.graalvm.compiler.hotspot.meta.HotSpotProviders;

import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;

import org.graalvm.compiler.hotspot.nodes.DimensionsNode;

import org.graalvm.compiler.hotspot.nodes.KlassBeingInitializedCheckNode;

import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyFixedNode;

import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyNode;

import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;

import org.graalvm.compiler.hotspot.word.KlassPointer;

import org.graalvm.compiler.nodes.ConstantNode;

import org.graalvm.compiler.nodes.DeoptimizeNode;

import org.graalvm.compiler.nodes.PiNode;

import org.graalvm.compiler.nodes.PrefetchAllocateNode;

import org.graalvm.compiler.nodes.SnippetAnchorNode;

import org.graalvm.compiler.nodes.StructuredGraph;

import org.graalvm.compiler.nodes.ValueNode;

import org.graalvm.compiler.nodes.debug.DynamicCounterNode;

import org.graalvm.compiler.nodes.debug.VerifyHeapNode;

import org.graalvm.compiler.nodes.extended.BranchProbabilityNode;

import org.graalvm.compiler.nodes.extended.ForeignCallNode;

import org.graalvm.compiler.nodes.extended.MembarNode;

import org.graalvm.compiler.nodes.java.DynamicNewArrayNode;

import org.graalvm.compiler.nodes.java.DynamicNewInstanceNode;

import org.graalvm.compiler.nodes.java.NewArrayNode;

import org.graalvm.compiler.nodes.java.NewInstanceNode;

import org.graalvm.compiler.nodes.java.NewMultiArrayNode;

import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;

import org.graalvm.compiler.nodes.spi.LoweringTool;

import org.graalvm.compiler.nodes.util.GraphUtil;

import org.graalvm.compiler.options.OptionValues;

import org.graalvm.compiler.replacements.ReplacementsUtil;

import org.graalvm.compiler.replacements.SnippetCounter;

import org.graalvm.compiler.replacements.SnippetCounter.Group;

import org.graalvm.compiler.replacements.SnippetTemplate;

import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;

import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;

import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;

import org.graalvm.compiler.replacements.Snippets;

import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode;

import org.graalvm.compiler.replacements.nodes.ZeroMemoryNode;

import org.graalvm.compiler.word.Word;

import jdk.internal.vm.compiler.word.LocationIdentity;

import jdk.internal.vm.compiler.word.WordFactory;

import jdk.vm.ci.code.CodeUtil;

import jdk.vm.ci.code.MemoryBarriers;

import jdk.vm.ci.code.Register;

import jdk.vm.ci.code.TargetDescription;

import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;

import jdk.vm.ci.meta.JavaKind;

import jdk.vm.ci.meta.ResolvedJavaType;

/**

 * Snippets used for implementing NEW, ANEWARRAY and NEWARRAY.

 */

public class NewObjectSnippets implements Snippets {

    enum ProfileContext {

        AllocatingMethod,

        InstanceOrArray,

        AllocatedType,

        AllocatedTypesInMethod,

        Total

    }

    @Fold

    static String createName(@Fold.InjectedParameter OptionValues options, String path, String typeContext) {

        switch (ProfileAllocationsContext.getValue(options)) {

            case AllocatingMethod:

                return "";

            case InstanceOrArray:

                return path;

            case AllocatedType:

            case AllocatedTypesInMethod:

                return typeContext;

            case Total:

                return "bytes";

            default:

                throw GraalError.shouldNotReachHere();

        }

    }

    @Fold

    static boolean doProfile(@Fold.InjectedParameter OptionValues options) {

        return ProfileAllocations.getValue(options);

    }

    @Fold

    static boolean withContext(@Fold.InjectedParameter OptionValues options) {

        ProfileContext context = ProfileAllocationsContext.getValue(options);

        return context == ProfileContext.AllocatingMethod || context == ProfileContext.AllocatedTypesInMethod;

    }

    protected static void profileAllocation(String path, long size, String typeContext) {

        if (doProfile(INJECTED_OPTIONVALUES)) {

            String name = createName(INJECTED_OPTIONVALUES, path, typeContext);

            boolean context = withContext(INJECTED_OPTIONVALUES);

            DynamicCounterNode.counter("number of bytes allocated", name, size, context);

            DynamicCounterNode.counter("number of allocations", name, 1, context);

        }

    }

    public static void emitPrefetchAllocate(Word address, boolean isArray) {

        if (allocatePrefetchStyle(INJECTED_VMCONFIG) > 0) {

            // Insert a prefetch for each allocation only on the fast-path

            // Generate several prefetch instructions.

            int lines = isArray ? allocatePrefetchLines(INJECTED_VMCONFIG) : allocateInstancePrefetchLines(INJECTED_VMCONFIG);

            int stepSize = allocatePrefetchStepSize(INJECTED_VMCONFIG);

            int distance = allocatePrefetchDistance(INJECTED_VMCONFIG);

            ExplodeLoopNode.explodeLoop();

            for (int i = 0; i < lines; i++) {

                PrefetchAllocateNode.prefetch(OffsetAddressNode.address(address, distance));

                distance += stepSize;

            }

        }

    }

    @Snippet

    public static Object allocateInstance(@ConstantParameter long size,

                    KlassPointer hub,

                    Word prototypeMarkWord,

                    @ConstantParameter boolean fillContents,

                    @ConstantParameter boolean emitMemoryBarrier,

                    @ConstantParameter Register threadRegister,

                    @ConstantParameter boolean constantSize,

                    @ConstantParameter String typeContext,

                    @ConstantParameter Counters counters) {

        return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, emitMemoryBarrier, threadRegister, constantSize, typeContext, counters));

    }

    public static Object allocateInstanceHelper(long size,

                    KlassPointer hub,

                    Word prototypeMarkWord,

                    boolean fillContents,

                    boolean emitMemoryBarrier,

                    Register threadRegister,

                    boolean constantSize,

                    String typeContext,

                    Counters counters) {

        Object result;

        Word thread = registerAsWord(threadRegister);

        Word top = readTlabTop(thread);

        Word end = readTlabEnd(thread);

        Word newTop = top.add(WordFactory.unsigned(size));

        if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {

            writeTlabTop(thread, newTop);

            emitPrefetchAllocate(newTop, false);

            result = formatObject(hub, size, top, prototypeMarkWord, fillContents, emitMemoryBarrier, constantSize, counters);

        } else {

            Counters theCounters = counters;

            if (theCounters != null && theCounters.stub != null) {

                theCounters.stub.inc();

            }

            result = newInstanceStub(hub);

        }

        profileAllocation("instance", size, typeContext);

        return verifyOop(result);

    }

    public static Object newInstanceStub(KlassPointer hub) {

        if (useNullAllocationStubs(INJECTED_VMCONFIG)) {

            return nonNullOrDeopt(newInstanceOrNull(NEW_INSTANCE_OR_NULL, hub));

        } else {

            return newInstance(NEW_INSTANCE, hub);

        }

    }

    @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)

    private static native Object newInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);

    @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)

    private static native Object newInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);

    @Snippet

    public static Object allocateInstancePIC(@ConstantParameter long size,

                    KlassPointer hub,

                    Word prototypeMarkWord,

                    @ConstantParameter boolean fillContents,

                    @ConstantParameter boolean emitMemoryBarrier,

                    @ConstantParameter Register threadRegister,

                    @ConstantParameter boolean constantSize,

                    @ConstantParameter String typeContext,

                    @ConstantParameter Counters counters) {

        // Klass must be initialized by the time the first instance is allocated, therefore we can

        // just load it from the corresponding cell and avoid the resolution check. We have to use a

        // fixed load though, to prevent it from floating above the initialization.

        KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);

        return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, picHub, prototypeMarkWord, fillContents, emitMemoryBarrier, threadRegister, constantSize, typeContext, counters));

    }

    @Snippet

    public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass,

                    @ConstantParameter boolean fillContents,

                    @ConstantParameter boolean emitMemoryBarrier,

                    @ConstantParameter Register threadRegister,

                    @ConstantParameter Counters counters) {

        if (probability(DEOPT_PROBABILITY, type == null)) {

            DeoptimizeNode.deopt(None, RuntimeConstraint);

        }

        Class<?> nonNullType = PiNode.piCastNonNullClass(type, SnippetAnchorNode.anchor());

        if (probability(DEOPT_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) {

            DeoptimizeNode.deopt(None, RuntimeConstraint);

        }

        return PiNode.piCastToSnippetReplaceeStamp(allocateInstanceDynamicHelper(type, fillContents, emitMemoryBarrier, threadRegister, counters, nonNullType));

    }

    private static Object allocateInstanceDynamicHelper(Class<?> type,

                    boolean fillContents,

                    boolean emitMemoryBarrier,

                    Register threadRegister,

                    Counters counters,

                    Class<?> nonNullType) {

        KlassPointer hub = ClassGetHubNode.readClass(nonNullType);

        if (probability(FAST_PATH_PROBABILITY, !hub.isNull())) {

            KlassPointer nonNullHub = ClassGetHubNode.piCastNonNull(hub, SnippetAnchorNode.anchor());

            if (probability(VERY_FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(nonNullHub))) {

                int layoutHelper = readLayoutHelper(nonNullHub);

                /*

                 * src/share/vm/oops/klass.hpp: For instances, layout helper is a positive number,

                 * the instance size. This size is already passed through align_object_size and

                 * scaled to bytes. The low order bit is set if instances of this class cannot be

                 * allocated using the fastpath.

                 */

                if (probability(FAST_PATH_PROBABILITY, (layoutHelper & 1) == 0)) {

                    Word prototypeMarkWord = nonNullHub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);

                    /*

                     * FIXME(je,ds): we should actually pass typeContext instead of "" but late

                     * binding of parameters is not yet supported by the GraphBuilderPlugin system.

                     */

                    return allocateInstanceHelper(layoutHelper, nonNullHub, prototypeMarkWord, fillContents, emitMemoryBarrier, threadRegister, false, "", counters);

                }

            } else {

                DeoptimizeNode.deopt(None, RuntimeConstraint);

            }

        }

        return dynamicNewInstanceStub(type);

    }

    /**

     * Maximum array length for which fast path allocation is used.

     */

    public static final int MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH = 0x00FFFFFF;

    @Snippet

    public static Object allocatePrimitiveArrayPIC(KlassPointer hub,

                    int length,

                    Word prototypeMarkWord,

                    @ConstantParameter int headerSize,

                    @ConstantParameter int log2ElementSize,

                    @ConstantParameter boolean fillContents,

                    @ConstantParameter boolean emitMemoryBarrier,

                    @ConstantParameter Register threadRegister,

                    @ConstantParameter boolean maybeUnroll,

                    @ConstantParameter String typeContext,

                    @ConstantParameter boolean supportsBulkZeroing,

                    @ConstantParameter Counters counters) {

        // Primitive array types are eagerly pre-resolved. We can use a floating load.

        KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);

        return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,

                        emitMemoryBarrier, threadRegister, maybeUnroll, typeContext, supportsBulkZeroing, counters);

    }

    @Snippet

    public static Object allocateArrayPIC(KlassPointer hub,

                    int length,

                    Word prototypeMarkWord,

                    @ConstantParameter int headerSize,

                    @ConstantParameter int log2ElementSize,

                    @ConstantParameter boolean fillContents,

                    @ConstantParameter boolean emitMemoryBarrier,

                    @ConstantParameter Register threadRegister,

                    @ConstantParameter boolean maybeUnroll,

                    @ConstantParameter String typeContext,

                    @ConstantParameter boolean supportsBulkZeroing,

                    @ConstantParameter Counters counters) {

        // Array type would be resolved by dominating resolution.

        KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);

        return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,

                        emitMemoryBarrier, threadRegister, maybeUnroll, typeContext, supportsBulkZeroing, counters);

    }

    @Snippet

    public static Object allocateArray(KlassPointer hub,

                    int length,

                    Word prototypeMarkWord,

                    @ConstantParameter int headerSize,

                    @ConstantParameter int log2ElementSize,

                    @ConstantParameter boolean fillContents,

                    @ConstantParameter boolean emitMemoryBarrier,

                    @ConstantParameter Register threadRegister,

                    @ConstantParameter boolean maybeUnroll,

                    @ConstantParameter String typeContext,

                    @ConstantParameter boolean supportsBulkZeroing,

                    @ConstantParameter Counters counters) {

        Object result = allocateArrayImpl(hub,

                        length,

                        prototypeMarkWord,

                        headerSize,

                        log2ElementSize,

                        fillContents,

                        emitMemoryBarrier, threadRegister,

                        maybeUnroll,

                        typeContext,

                        supportsBulkZeroing,

                        counters);

        return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);

    }

    /**

     * When allocating on the slow path, determines whether to use a version of the runtime call

     * that returns {@code null} on a failed allocation instead of raising an OutOfMemoryError.

     */

    @Fold

    static boolean useNullAllocationStubs(@InjectedParameter GraalHotSpotVMConfig config) {

        return config.areNullAllocationStubsAvailable();

    }

    private static Object allocateArrayImpl(KlassPointer hub,

                    int length,

                    Word prototypeMarkWord,

                    int headerSize,

                    int log2ElementSize,

                    boolean fillContents,

                    boolean emitMemoryBarrier,

                    Register threadRegister,

                    boolean maybeUnroll,

                    String typeContext,

                    boolean supportsBulkZeroing,

                    Counters counters) {

        Object result;

        long allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize);

        Word thread = registerAsWord(threadRegister);

        Word top = readTlabTop(thread);

        Word end = readTlabEnd(thread);

        Word newTop = top.add(WordFactory.unsigned(allocationSize));

        if (probability(FREQUENT_PROBABILITY, belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&

                        probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {

            writeTlabTop(thread, newTop);

            emitPrefetchAllocate(newTop, true);

            Counters theCounters = counters;

            if (theCounters != null && theCounters.arrayLoopInit != null) {

                theCounters.arrayLoopInit.inc();

            }

            result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, emitMemoryBarrier, maybeUnroll, supportsBulkZeroing, counters);

        } else {

            result = newArrayStub(hub, length);

        }

        profileAllocation("array", allocationSize, typeContext);

        return result;

    }

    public static Object newArrayStub(KlassPointer hub, int length) {

        if (useNullAllocationStubs(INJECTED_VMCONFIG)) {

            return nonNullOrDeopt(newArrayOrNull(NEW_ARRAY_OR_NULL, hub, length));

        } else {

            return newArray(NEW_ARRAY, hub, length);

        }

    }

    @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)

    private static native Object newArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length);

    @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)

    private static native Object newArrayOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length);

    /**

     * New dynamic array stub that throws an {@link OutOfMemoryError} on allocation failure.

     */

    public static final ForeignCallDescriptor DYNAMIC_NEW_INSTANCE = new ForeignCallDescriptor("dynamic_new_instance", Object.class, Class.class);

    /**

     * New dynamic array stub that returns null on allocation failure.

     */

    public static final ForeignCallDescriptor DYNAMIC_NEW_INSTANCE_OR_NULL = new ForeignCallDescriptor("dynamic_new_instance_or_null", Object.class, Class.class);

    public static Object dynamicNewInstanceStub(Class<?> elementType) {

        if (useNullAllocationStubs(INJECTED_VMCONFIG)) {

            return nonNullOrDeopt(dynamicNewInstanceOrNull(DYNAMIC_NEW_INSTANCE_OR_NULL, elementType));

        } else {

            return dynamicNewInstance(DYNAMIC_NEW_INSTANCE, elementType);

        }

    }

    /**

     * Deoptimizes if {@code obj == null} otherwise returns {@code obj}.

     */

    private static Object nonNullOrDeopt(Object obj) {

        if (BranchProbabilityNode.probability(BranchProbabilityNode.DEOPT_PROBABILITY, obj == null)) {

            DeoptimizeNode.deopt(None, RuntimeConstraint);

        }

        return obj;

    }