--- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/arraycopy/ArrayCopySnippets.java Mon Dec 10 16:49:54 2018 -0500
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,573 +0,0 @@
-/*
- * Copyright (c) 2011, 2016, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-
-package org.graalvm.compiler.hotspot.replacements.arraycopy;
-
-import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
-import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_METAACCESS;
-import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.KLASS_SUPER_CHECK_OFFSET_LOCATION;
-import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION;
-import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayClassElementOffset;
-import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.getArrayBaseOffset;
-import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadHub;
-import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
-import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.superCheckOffsetOffset;
-import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
-import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.LIKELY_PROBABILITY;
-import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY;
-import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
-import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
-
-import java.lang.reflect.Method;
-import java.util.EnumMap;
-
-import jdk.internal.vm.compiler.collections.UnmodifiableEconomicMap;
-import org.graalvm.compiler.api.directives.GraalDirectives;
-import org.graalvm.compiler.api.replacements.Fold;
-import org.graalvm.compiler.api.replacements.Snippet;
-import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
-import org.graalvm.compiler.debug.DebugHandlersFactory;
-import org.graalvm.compiler.debug.GraalError;
-import org.graalvm.compiler.graph.Node;
-import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
-import org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil;
-import org.graalvm.compiler.hotspot.word.KlassPointer;
-import org.graalvm.compiler.nodes.CallTargetNode;
-import org.graalvm.compiler.nodes.DeoptimizeNode;
-import org.graalvm.compiler.nodes.Invoke;
-import org.graalvm.compiler.nodes.InvokeNode;
-import org.graalvm.compiler.nodes.NamedLocationIdentity;
-import org.graalvm.compiler.nodes.NodeView;
-import org.graalvm.compiler.nodes.PiNode;
-import org.graalvm.compiler.nodes.StructuredGraph;
-import org.graalvm.compiler.nodes.ValueNode;
-import org.graalvm.compiler.nodes.extended.RawLoadNode;
-import org.graalvm.compiler.nodes.extended.RawStoreNode;
-import org.graalvm.compiler.nodes.java.ArrayLengthNode;
-import org.graalvm.compiler.nodes.spi.LoweringTool;
-import org.graalvm.compiler.nodes.type.StampTool;
-import org.graalvm.compiler.nodes.util.GraphUtil;
-import org.graalvm.compiler.options.OptionValues;
-import org.graalvm.compiler.replacements.ReplacementsUtil;
-import org.graalvm.compiler.replacements.SnippetCounter;
-import org.graalvm.compiler.replacements.SnippetCounter.Group;
-import org.graalvm.compiler.replacements.SnippetIntegerHistogram;
-import org.graalvm.compiler.replacements.SnippetTemplate;
-import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
-import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
-import org.graalvm.compiler.replacements.Snippets;
-import org.graalvm.compiler.replacements.nodes.BasicArrayCopyNode;
-import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode;
-import org.graalvm.compiler.word.Word;
-import jdk.internal.vm.compiler.word.LocationIdentity;
-import jdk.internal.vm.compiler.word.WordFactory;
-
-import jdk.vm.ci.code.TargetDescription;
-import jdk.vm.ci.meta.DeoptimizationAction;
-import jdk.vm.ci.meta.DeoptimizationReason;
-import jdk.vm.ci.meta.JavaKind;
-import jdk.vm.ci.meta.ResolvedJavaMethod;
-import jdk.vm.ci.meta.ResolvedJavaType;
-
-public class ArrayCopySnippets implements Snippets {
-
- private enum ArrayCopyTypeCheck {
- UNDEFINED_ARRAY_TYPE_CHECK,
- // either we know that both objects are arrays and have the same type,
- // or we apply generic array copy snippet, which enforces type check
- NO_ARRAY_TYPE_CHECK,
- // can be used when we know that one of the objects is a primitive array
- HUB_BASED_ARRAY_TYPE_CHECK,
- // can be used when we know that one of the objects is an object array
- LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK
- }
-
- @Snippet
- public static void arraycopyZeroLengthSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
- @ConstantParameter Counters counters) {
- Object nonNullSrc = GraalDirectives.guardingNonNull(src);
- Object nonNullDest = GraalDirectives.guardingNonNull(dest);
- checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
- checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
- counters.zeroLengthStaticCounter.inc();
- }
-
- @Snippet
- public static void arraycopyExactSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
- @ConstantParameter JavaKind elementKind, @ConstantParameter SnippetCounter elementKindCounter, @ConstantParameter SnippetCounter elementKindCopiedCounter,
- @ConstantParameter Counters counters) {
- Object nonNullSrc = GraalDirectives.guardingNonNull(src);
- Object nonNullDest = GraalDirectives.guardingNonNull(dest);
- checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
- checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
- incrementLengthCounter(length, counters);
-
- elementKindCounter.inc();
- elementKindCopiedCounter.add(length);
- ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind);
- }
-
- @Snippet
- public static void arraycopyUnrolledSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
- @ConstantParameter JavaKind elementKind, @ConstantParameter int unrolledLength, @ConstantParameter Counters counters) {
- Object nonNullSrc = GraalDirectives.guardingNonNull(src);
- Object nonNullDest = GraalDirectives.guardingNonNull(dest);
- checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
- checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
- incrementLengthCounter(length, counters);
-
- unrolledArraycopyWork(nonNullSrc, srcPos, nonNullDest, destPos, unrolledLength, elementKind);
- }
-
- @Snippet
- public static void arraycopyCheckcastSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
- @ConstantParameter Counters counters, @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind) {
- Object nonNullSrc = GraalDirectives.guardingNonNull(src);
- Object nonNullDest = GraalDirectives.guardingNonNull(dest);
- checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
- checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
- incrementLengthCounter(length, counters);
-
- ArrayCopyWithSlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind);
- }
-
- @Snippet
- public static void arraycopyGenericSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, @ConstantParameter Counters counters,
- @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind) {
- Object nonNullSrc = GraalDirectives.guardingNonNull(src);
- Object nonNullDest = GraalDirectives.guardingNonNull(dest);
- checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
- checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
- incrementLengthCounter(length, counters);
-
- ArrayCopyWithSlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind);
- }
-
- @Snippet
- public static void arraycopyNativeSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
- // all checks are done in the native method, so no need to emit additional checks here
- incrementLengthCounter(length, counters);
- counters.systemArraycopyCounter.inc();
- counters.systemArraycopyCopiedCounter.add(length);
-
- System.arraycopy(src, srcPos, dest, destPos, length);
- }
-
- @Fold
- static LocationIdentity getArrayLocation(JavaKind kind) {
- return NamedLocationIdentity.getArrayLocation(kind);
- }
-
- private static void unrolledArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, JavaKind elementKind) {
- int scale = HotSpotReplacementsUtil.arrayIndexScale(INJECTED_METAACCESS, elementKind);
- int arrayBaseOffset = getArrayBaseOffset(INJECTED_METAACCESS, elementKind);
- LocationIdentity arrayLocation = getArrayLocation(elementKind);
-
- long sourceOffset = arrayBaseOffset + (long) srcPos * scale;
- long destOffset = arrayBaseOffset + (long) destPos * scale;
- long position = 0;
- long delta = scale;
- if (probability(NOT_FREQUENT_PROBABILITY, nonNullSrc == nonNullDest && srcPos < destPos)) {
- // bad aliased case so we need to copy the array from back to front
- position = (long) (length - 1) * scale;
- delta = -delta;
- }
-
- // the length was already checked before - we can emit unconditional instructions
- ExplodeLoopNode.explodeLoop();
- for (int iteration = 0; iteration < length; iteration++) {
- Object value = RawLoadNode.load(nonNullSrc, sourceOffset + position, elementKind, arrayLocation);
- RawStoreNode.storeObject(nonNullDest, destOffset + position, value, elementKind, arrayLocation, false);
- position += delta;
- }
- }
-
- @Snippet(allowPartialIntrinsicArgumentMismatch = true)
- public static void checkcastArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
- if (probability(FREQUENT_PROBABILITY, length > 0)) {
- Object nonNullSrc = PiNode.asNonNullObject(src);
- Object nonNullDest = PiNode.asNonNullObject(dest);
- KlassPointer srcKlass = loadHub(nonNullSrc);
- KlassPointer destKlass = loadHub(nonNullDest);
- if (probability(LIKELY_PROBABILITY, srcKlass == destKlass)) {
- // no storecheck required.
- counters.objectCheckcastSameTypeCounter.inc();
- counters.objectCheckcastSameTypeCopiedCounter.add(length);
- ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length);
- } else {
- KlassPointer destElemKlass = destKlass.readKlassPointer(arrayClassElementOffset(INJECTED_VMCONFIG), OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION);
- Word superCheckOffset = WordFactory.signed(destElemKlass.readInt(superCheckOffsetOffset(INJECTED_VMCONFIG), KLASS_SUPER_CHECK_OFFSET_LOCATION));
-
- counters.objectCheckcastDifferentTypeCounter.inc();
- counters.objectCheckcastDifferentTypeCopiedCounter.add(length);
-
- int copiedElements = CheckcastArrayCopyCallNode.checkcastArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, superCheckOffset, destElemKlass, false);
- if (probability(SLOW_PATH_PROBABILITY, copiedElements != 0)) {
- /*
- * the stub doesn't throw the ArrayStoreException, but returns the number of
- * copied elements (xor'd with -1).
- */
- copiedElements ^= -1;
- System.arraycopy(nonNullSrc, srcPos + copiedElements, nonNullDest, destPos + copiedElements, length - copiedElements);
- }
- }
- }
- }
-
- @Snippet(allowPartialIntrinsicArgumentMismatch = true)
- public static void genericArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
- // The length > 0 check should not be placed here because generic array copy stub should
- // enforce type check. This is fine performance-wise because this snippet is rarely used.
- counters.genericArraycopyDifferentTypeCounter.inc();
- counters.genericArraycopyDifferentTypeCopiedCounter.add(length);
- int copiedElements = GenericArrayCopyCallNode.genericArraycopy(src, srcPos, dest, destPos, length);
- if (probability(SLOW_PATH_PROBABILITY, copiedElements != 0)) {
- /*
- * the stub doesn't throw the ArrayStoreException, but returns the number of copied
- * elements (xor'd with -1).
- */
- copiedElements ^= -1;
- System.arraycopy(src, srcPos + copiedElements, dest, destPos + copiedElements, length - copiedElements);
- }
- }
-
- private static void incrementLengthCounter(int length, Counters counters) {
- counters.lengthHistogram.inc(length);
- }
-
- private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length, Counters counters) {
- if (probability(SLOW_PATH_PROBABILITY, srcPos < 0) ||
- probability(SLOW_PATH_PROBABILITY, destPos < 0) ||
- probability(SLOW_PATH_PROBABILITY, length < 0) ||
- probability(SLOW_PATH_PROBABILITY, srcPos > ArrayLengthNode.arrayLength(src) - length) ||
- probability(SLOW_PATH_PROBABILITY, destPos > ArrayLengthNode.arrayLength(dest) - length)) {
- counters.checkAIOOBECounter.inc();
- DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
- }
- counters.checkSuccessCounter.inc();
- }
-
- private static void checkArrayTypes(Object nonNullSrc, Object nonNullDest, ArrayCopyTypeCheck arrayTypeCheck) {
- if (arrayTypeCheck == ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK) {
- // nothing to do
- } else if (arrayTypeCheck == ArrayCopyTypeCheck.HUB_BASED_ARRAY_TYPE_CHECK) {
- KlassPointer srcHub = loadHub(nonNullSrc);
- KlassPointer destHub = loadHub(nonNullDest);
- if (probability(SLOW_PATH_PROBABILITY, srcHub != destHub)) {
- DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
- }
- } else if (arrayTypeCheck == ArrayCopyTypeCheck.LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK) {
- KlassPointer srcHub = loadHub(nonNullSrc);
- KlassPointer destHub = loadHub(nonNullDest);
- if (probability(SLOW_PATH_PROBABILITY, readLayoutHelper(srcHub) != readLayoutHelper(destHub))) {
- DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
- }
- } else {
- ReplacementsUtil.staticAssert(false, "unknown array type check");
- }
- }
-
- static class Counters {
- final SnippetCounter checkSuccessCounter;
- final SnippetCounter checkAIOOBECounter;
-
- final SnippetCounter zeroLengthStaticCounter;
- final SnippetIntegerHistogram lengthHistogram;
-
- final SnippetCounter systemArraycopyCounter;
- final SnippetCounter systemArraycopyCopiedCounter;
-
- final SnippetCounter genericArraycopyDifferentTypeCopiedCounter;
- final SnippetCounter genericArraycopyDifferentTypeCounter;
-
- final SnippetCounter objectCheckcastSameTypeCopiedCounter;
- final SnippetCounter objectCheckcastSameTypeCounter;
- final SnippetCounter objectCheckcastDifferentTypeCopiedCounter;
- final SnippetCounter objectCheckcastDifferentTypeCounter;
-
- final EnumMap<JavaKind, SnippetCounter> arraycopyCallCounters = new EnumMap<>(JavaKind.class);
- final EnumMap<JavaKind, SnippetCounter> arraycopyCallCopiedCounters = new EnumMap<>(JavaKind.class);
-
- Counters(SnippetCounter.Group.Factory factory) {
- final Group checkCounters = factory.createSnippetCounterGroup("System.arraycopy checkInputs");
- final Group callCounters = factory.createSnippetCounterGroup("System.arraycopy calls");
- final Group copiedElementsCounters = factory.createSnippetCounterGroup("System.arraycopy copied elements");
- final Group lengthCounters = factory.createSnippetCounterGroup("System.arraycopy with 0-length");
-
- checkSuccessCounter = new SnippetCounter(checkCounters, "checkSuccess", "checkSuccess");
- checkAIOOBECounter = new SnippetCounter(checkCounters, "checkAIOOBE", "checkAIOOBE");
-
- zeroLengthStaticCounter = new SnippetCounter(lengthCounters, "0-length copy static", "calls where the length is statically 0");
- lengthHistogram = new SnippetIntegerHistogram(lengthCounters, 2, "length", "length");
-
- systemArraycopyCounter = new SnippetCounter(callCounters, "native System.arraycopy", "JNI-based System.arraycopy call");
- systemArraycopyCopiedCounter = new SnippetCounter(copiedElementsCounters, "native System.arraycopy", "JNI-based System.arraycopy call");
-
- genericArraycopyDifferentTypeCounter = new SnippetCounter(callCounters, "generic[] stub", "generic arraycopy stub");
- genericArraycopyDifferentTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "generic[] stub", "generic arraycopy stub");
-
- objectCheckcastSameTypeCounter = new SnippetCounter(callCounters, "checkcast object[] (same-type)", "checkcast object[] stub but src.klass == dest.klass Object[] arrays");
- objectCheckcastSameTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "checkcast object[] (same-type)", "checkcast object[] stub but src.klass == dest.klass Object[] arrays");
- objectCheckcastDifferentTypeCounter = new SnippetCounter(callCounters, "checkcast object[] (store-check)", "checkcast object[] stub with store check");
- objectCheckcastDifferentTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "checkcast object[] (store-check)", "checkcast object[] stub with store check");
-
- createArraycopyCounter(JavaKind.Byte, callCounters, copiedElementsCounters);
- createArraycopyCounter(JavaKind.Boolean, callCounters, copiedElementsCounters);
- createArraycopyCounter(JavaKind.Char, callCounters, copiedElementsCounters);
- createArraycopyCounter(JavaKind.Short, callCounters, copiedElementsCounters);
- createArraycopyCounter(JavaKind.Int, callCounters, copiedElementsCounters);
- createArraycopyCounter(JavaKind.Long, callCounters, copiedElementsCounters);
- createArraycopyCounter(JavaKind.Float, callCounters, copiedElementsCounters);
- createArraycopyCounter(JavaKind.Double, callCounters, copiedElementsCounters);
- createArraycopyCounter(JavaKind.Object, callCounters, copiedElementsCounters);
- }
-
- void createArraycopyCounter(JavaKind kind, Group counters, Group copiedCounters) {
- arraycopyCallCounters.put(kind, new SnippetCounter(counters, kind + "[] stub", "arraycopy call for " + kind + "[] arrays"));
- arraycopyCallCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[] stub", "arraycopy call for " + kind + "[] arrays"));
- }
- }
-
- public static class Templates extends SnippetTemplate.AbstractTemplates {
- private final SnippetInfo arraycopyGenericSnippet = snippet("arraycopyGenericSnippet");
- private final SnippetInfo arraycopyUnrolledSnippet = snippet("arraycopyUnrolledSnippet");
- private final SnippetInfo arraycopyExactSnippet = snippet("arraycopyExactSnippet");
- private final SnippetInfo arraycopyZeroLengthSnippet = snippet("arraycopyZeroLengthSnippet");
- private final SnippetInfo arraycopyCheckcastSnippet = snippet("arraycopyCheckcastSnippet");
- private final SnippetInfo arraycopyNativeSnippet = snippet("arraycopyNativeSnippet");
-
- private final SnippetInfo checkcastArraycopyWithSlowPathWork = snippet("checkcastArraycopyWithSlowPathWork");
- private final SnippetInfo genericArraycopyWithSlowPathWork = snippet("genericArraycopyWithSlowPathWork");
-
- private ResolvedJavaMethod originalArraycopy;
- private final Counters counters;
-
- public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target) {
- super(options, factories, providers, providers.getSnippetReflection(), target);
- this.counters = new Counters(factory);
- }
-
- protected SnippetInfo snippet(String methodName) {
- SnippetInfo info = snippet(ArrayCopySnippets.class, methodName, LocationIdentity.any());
- info.setOriginalMethod(originalArraycopy());
- return info;
- }
-
- public void lower(ArrayCopyNode arraycopy, LoweringTool tool) {
- JavaKind elementKind = selectComponentKind(arraycopy);
- SnippetInfo snippetInfo;
- ArrayCopyTypeCheck arrayTypeCheck;
-
- ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp(NodeView.DEFAULT));
- ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp(NodeView.DEFAULT));
- if (!canBeArray(srcType) || !canBeArray(destType)) {
- // at least one of the objects is definitely not an array - use the native call
- // right away as the copying will fail anyways
- snippetInfo = arraycopyNativeSnippet;
- arrayTypeCheck = ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK;
- } else {
- ResolvedJavaType srcComponentType = srcType == null ? null : srcType.getComponentType();
- ResolvedJavaType destComponentType = destType == null ? null : destType.getComponentType();
-
- if (arraycopy.isExact()) {
- // there is a sufficient type match - we don't need any additional type checks
- snippetInfo = arraycopyExactSnippet;
- arrayTypeCheck = ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK;
- } else if (srcComponentType == null && destComponentType == null) {
- // we don't know anything about the types - use the generic copying
- snippetInfo = arraycopyGenericSnippet;
- // no need for additional type check to avoid duplicated work
- arrayTypeCheck = ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK;
- } else if (srcComponentType != null && destComponentType != null) {
- if (!srcComponentType.isPrimitive() && !destComponentType.isPrimitive()) {
- // it depends on the array content if the copy succeeds - we need
- // a type check for every store
- snippetInfo = arraycopyCheckcastSnippet;
- arrayTypeCheck = ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK;
- } else {
- // one object is an object array, the other one is a primitive array.
- // this copy will always fail - use the native call right away
- assert !srcComponentType.equals(destComponentType) : "must be handled by arraycopy.isExact()";
- snippetInfo = arraycopyNativeSnippet;
- arrayTypeCheck = ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK;
- }
- } else {
- ResolvedJavaType nonNullComponentType = srcComponentType != null ? srcComponentType : destComponentType;
- if (nonNullComponentType.isPrimitive()) {
- // one involved object is a primitive array - it is sufficient to directly
- // compare the hub.
- snippetInfo = arraycopyExactSnippet;
- arrayTypeCheck = ArrayCopyTypeCheck.HUB_BASED_ARRAY_TYPE_CHECK;
- elementKind = nonNullComponentType.getJavaKind();
- } else {
- // one involved object is an object array - the other array's element type
- // may be primitive or object, hence we compare the layout helper.
- snippetInfo = arraycopyCheckcastSnippet;
- arrayTypeCheck = ArrayCopyTypeCheck.LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK;
- }
- }
- }
-
- // a few special cases that are easier to handle when all other variables already have a
- // value
- if (snippetInfo != arraycopyNativeSnippet && snippetInfo != arraycopyGenericSnippet && arraycopy.getLength().isConstant() && arraycopy.getLength().asJavaConstant().asLong() == 0) {
- // Copying 0 element between object arrays with conflicting types will not throw an
- // exception - once we pass the preliminary element type checks that we are not
- // mixing arrays of different basic types, ArrayStoreException is only thrown when
- // an *astore would have thrown it. Therefore, copying null between object arrays
- // with conflicting types will also succeed (we do not optimize for such case here).
- snippetInfo = arraycopyZeroLengthSnippet;
- } else if (snippetInfo == arraycopyExactSnippet && shouldUnroll(arraycopy.getLength())) {
- snippetInfo = arraycopyUnrolledSnippet;
- }
-
- // create the snippet
- Arguments args = new Arguments(snippetInfo, arraycopy.graph().getGuardsStage(), tool.getLoweringStage());
- args.add("src", arraycopy.getSource());
- args.add("srcPos", arraycopy.getSourcePosition());
- args.add("dest", arraycopy.getDestination());
- args.add("destPos", arraycopy.getDestinationPosition());
- args.add("length", arraycopy.getLength());
- if (snippetInfo != arraycopyNativeSnippet) {
- assert arrayTypeCheck != ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK;
- args.addConst("arrayTypeCheck", arrayTypeCheck);
- }
- if (snippetInfo == arraycopyUnrolledSnippet) {
- args.addConst("elementKind", elementKind != null ? elementKind : JavaKind.Illegal);
- args.addConst("unrolledLength", arraycopy.getLength().asJavaConstant().asInt());
- }
- if (snippetInfo == arraycopyExactSnippet) {
- assert elementKind != null;
- args.addConst("elementKind", elementKind);
- args.addConst("elementKindCounter", counters.arraycopyCallCounters.get(elementKind));
- args.addConst("elementKindCopiedCounter", counters.arraycopyCallCopiedCounters.get(elementKind));
- }
- args.addConst("counters", counters);
- if (snippetInfo == arraycopyCheckcastSnippet) {
- args.addConst("workSnippet", checkcastArraycopyWithSlowPathWork);
- args.addConst("elementKind", JavaKind.Illegal);
- }
- if (snippetInfo == arraycopyGenericSnippet) {
- args.addConst("workSnippet", genericArraycopyWithSlowPathWork);
- args.addConst("elementKind", JavaKind.Illegal);
- }
-
- instantiate(args, arraycopy);
- }
-
- public void lower(ArrayCopyWithSlowPathNode arraycopy, LoweringTool tool) {
- StructuredGraph graph = arraycopy.graph();
- if (!graph.getGuardsStage().areFrameStatesAtDeopts()) {
- // if an arraycopy contains a slow path, we can't lower it right away
- return;
- }
-
- SnippetInfo snippetInfo = arraycopy.getSnippet();
- Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage());
- args.add("src", arraycopy.getSource());
- args.add("srcPos", arraycopy.getSourcePosition());
- args.add("dest", arraycopy.getDestination());
- args.add("destPos", arraycopy.getDestinationPosition());
- args.add("length", arraycopy.getLength());
- args.addConst("counters", counters);
- instantiate(args, arraycopy);
- }
-
- private static boolean canBeArray(ResolvedJavaType type) {
- return type == null || type.isJavaLangObject() || type.isArray();
- }
-
- public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy) {
- ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp(NodeView.DEFAULT));
- ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp(NodeView.DEFAULT));
-
- if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) {
- return null;
- }
- if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) {
- return null;
- }
- if (!arraycopy.isExact()) {
- return null;
- }
- return srcType.getComponentType().getJavaKind();
- }
-
- private static boolean shouldUnroll(ValueNode length) {
- return length.isConstant() && length.asJavaConstant().asInt() <= 8 && length.asJavaConstant().asInt() != 0;
- }
-
- /**
- * Instantiate the snippet template and fix up the FrameState of any Invokes of
- * System.arraycopy and propagate the captured bci in the ArrayCopySlowPathNode.
- *
- * @param args
- * @param arraycopy
- */
- private void instantiate(Arguments args, BasicArrayCopyNode arraycopy) {
- StructuredGraph graph = arraycopy.graph();
- SnippetTemplate template = template(arraycopy, args);
- UnmodifiableEconomicMap<Node, Node> replacements = template.instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args, false);
- for (Node originalNode : replacements.getKeys()) {
- if (originalNode instanceof Invoke) {
- Invoke invoke = (Invoke) replacements.get(originalNode);
- assert invoke.asNode().graph() == graph;
- CallTargetNode call = invoke.callTarget();
-
- if (!call.targetMethod().equals(originalArraycopy)) {
- throw new GraalError("unexpected invoke %s in snippet", call.targetMethod());
- }
- // Here we need to fix the bci of the invoke
- InvokeNode newInvoke = graph.add(new InvokeNode(invoke.callTarget(), arraycopy.getBci()));
- if (arraycopy.stateDuring() != null) {
- newInvoke.setStateDuring(arraycopy.stateDuring());
- } else {
- assert arraycopy.stateAfter() != null : arraycopy;
- newInvoke.setStateAfter(arraycopy.stateAfter());
- }
- graph.replaceFixedWithFixed((InvokeNode) invoke.asNode(), newInvoke);
- } else if (originalNode instanceof ArrayCopyWithSlowPathNode) {
- ArrayCopyWithSlowPathNode slowPath = (ArrayCopyWithSlowPathNode) replacements.get(originalNode);
- assert arraycopy.stateAfter() != null : arraycopy;
- assert slowPath.stateAfter() == arraycopy.stateAfter();
- slowPath.setBci(arraycopy.getBci());
- }
- }
- GraphUtil.killCFG(arraycopy);
- }
-
- private ResolvedJavaMethod originalArraycopy() throws GraalError {
- if (originalArraycopy == null) {
- Method method;
- try {
- method = System.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class);
- } catch (NoSuchMethodException | SecurityException e) {
- throw new GraalError(e);
- }
- originalArraycopy = providers.getMetaAccess().lookupJavaMethod(method);
- }
- return originalArraycopy;
- }
- }
-}