--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/hotspot/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/WriteBarrierSnippets.java Thu Feb 16 15:46:09 2017 -0800
@@ -0,0 +1,556 @@
+/*
+ * Copyright (c) 2012, 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.compiler.hotspot.replacements;
+
+import static org.graalvm.compiler.core.common.GraalOptions.SnippetCounters;
+import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayIndexScale;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.cardTableShift;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.dirtyCardValue;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1CardQueueBufferOffset;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1CardQueueIndexOffset;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1SATBQueueBufferOffset;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1SATBQueueIndexOffset;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1SATBQueueMarkingOffset;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1YoungCardValue;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOops;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize;
+import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
+import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.LIKELY_PROBABILITY;
+import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY;
+import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
+import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
+import static jdk.vm.ci.code.MemoryBarriers.STORE_LOAD;
+
+import org.graalvm.compiler.api.replacements.Snippet;
+import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
+import org.graalvm.compiler.core.common.GraalOptions;
+import org.graalvm.compiler.core.common.LocationIdentity;
+import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
+import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
+import org.graalvm.compiler.graph.Node.NodeIntrinsic;
+import org.graalvm.compiler.hotspot.CompressEncoding;
+import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
+import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
+import org.graalvm.compiler.hotspot.nodes.CompressionNode;
+import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePostWriteBarrier;
+import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePreWriteBarrier;
+import org.graalvm.compiler.hotspot.nodes.G1PostWriteBarrier;
+import org.graalvm.compiler.hotspot.nodes.G1PreWriteBarrier;
+import org.graalvm.compiler.hotspot.nodes.G1ReferentFieldReadBarrier;
+import org.graalvm.compiler.hotspot.nodes.GetObjectAddressNode;
+import org.graalvm.compiler.hotspot.nodes.GraalHotSpotVMConfigNode;
+import org.graalvm.compiler.hotspot.nodes.SerialArrayRangeWriteBarrier;
+import org.graalvm.compiler.hotspot.nodes.SerialWriteBarrier;
+import org.graalvm.compiler.hotspot.nodes.type.NarrowOopStamp;
+import org.graalvm.compiler.nodes.NamedLocationIdentity;
+import org.graalvm.compiler.nodes.StructuredGraph;
+import org.graalvm.compiler.nodes.ValueNode;
+import org.graalvm.compiler.nodes.extended.FixedValueAnchorNode;
+import org.graalvm.compiler.nodes.extended.ForeignCallNode;
+import org.graalvm.compiler.nodes.extended.MembarNode;
+import org.graalvm.compiler.nodes.extended.NullCheckNode;
+import org.graalvm.compiler.nodes.memory.HeapAccess.BarrierType;
+import org.graalvm.compiler.nodes.memory.address.AddressNode;
+import org.graalvm.compiler.nodes.memory.address.AddressNode.Address;
+import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
+import org.graalvm.compiler.nodes.spi.LoweringTool;
+import org.graalvm.compiler.replacements.Log;
+import org.graalvm.compiler.replacements.SnippetCounter;
+import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
+import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
+import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
+import org.graalvm.compiler.replacements.Snippets;
+import org.graalvm.compiler.replacements.nodes.DirectObjectStoreNode;
+import org.graalvm.compiler.replacements.nodes.DirectStoreNode;
+import org.graalvm.compiler.word.Pointer;
+import org.graalvm.compiler.word.Unsigned;
+import org.graalvm.compiler.word.Word;
+
+import jdk.vm.ci.code.Register;
+import jdk.vm.ci.code.TargetDescription;
+import jdk.vm.ci.meta.JavaKind;
+
+public class WriteBarrierSnippets implements Snippets {
+
+ private static final SnippetCounter.Group countersWriteBarriers = SnippetCounters.getValue() ? new SnippetCounter.Group("WriteBarriers") : null;
+ private static final SnippetCounter serialWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "serialWriteBarrier", "Number of Serial Write Barriers");
+ private static final SnippetCounter g1AttemptedPreWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1AttemptedPreWriteBarrier", "Number of attempted G1 Pre Write Barriers");
+ private static final SnippetCounter g1EffectivePreWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1EffectivePreWriteBarrier", "Number of effective G1 Pre Write Barriers");
+ private static final SnippetCounter g1ExecutedPreWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1ExecutedPreWriteBarrier", "Number of executed G1 Pre Write Barriers");
+ private static final SnippetCounter g1AttemptedPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1AttemptedPostWriteBarrier", "Number of attempted G1 Post Write Barriers");
+ private static final SnippetCounter g1EffectiveAfterXORPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1EffectiveAfterXORPostWriteBarrier",
+ "Number of effective G1 Post Write Barriers (after passing the XOR test)");
+ private static final SnippetCounter g1EffectiveAfterNullPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1EffectiveAfterNullPostWriteBarrier",
+ "Number of effective G1 Post Write Barriers (after passing the NULL test)");
+ private static final SnippetCounter g1ExecutedPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1ExecutedPostWriteBarrier", "Number of executed G1 Post Write Barriers");
+
+ public static final LocationIdentity GC_CARD_LOCATION = NamedLocationIdentity.mutable("GC-Card");
+ public static final LocationIdentity GC_LOG_LOCATION = NamedLocationIdentity.mutable("GC-Log");
+ public static final LocationIdentity GC_INDEX_LOCATION = NamedLocationIdentity.mutable("GC-Index");
+
+ private static void serialWriteBarrier(Pointer ptr) {
+ serialWriteBarrierCounter.inc();
+ final long startAddress = GraalHotSpotVMConfigNode.cardTableAddress();
+ Word base = (Word) ptr.unsignedShiftRight(cardTableShift(INJECTED_VMCONFIG));
+ if (((int) startAddress) == startAddress && GraalHotSpotVMConfigNode.isCardTableAddressConstant()) {
+ base.writeByte((int) startAddress, (byte) 0, GC_CARD_LOCATION);
+ } else {
+ base.writeByte(Word.unsigned(startAddress), (byte) 0, GC_CARD_LOCATION);
+ }
+ }
+
+ @Snippet
+ public static void serialImpreciseWriteBarrier(Object object) {
+ serialWriteBarrier(Word.objectToTrackedPointer(object));
+ }
+
+ @Snippet
+ public static void serialPreciseWriteBarrier(Address address) {
+ serialWriteBarrier(Word.fromAddress(address));
+ }
+
+ @Snippet
+ public static void serialArrayRangeWriteBarrier(Object object, int startIndex, int length) {
+ if (length == 0) {
+ return;
+ }
+ Object dest = FixedValueAnchorNode.getObject(object);
+ int cardShift = cardTableShift(INJECTED_VMCONFIG);
+ final long cardStart = GraalHotSpotVMConfigNode.cardTableAddress();
+ final int scale = arrayIndexScale(JavaKind.Object);
+ int header = arrayBaseOffset(JavaKind.Object);
+ long dstAddr = GetObjectAddressNode.get(dest);
+ long start = (dstAddr + header + (long) startIndex * scale) >>> cardShift;
+ long end = (dstAddr + header + ((long) startIndex + length - 1) * scale) >>> cardShift;
+ long count = end - start + 1;
+ while (count-- > 0) {
+ DirectStoreNode.storeBoolean((start + cardStart) + count, false, JavaKind.Boolean);
+ }
+ }
+
+ @Snippet
+ public static void g1PreWriteBarrier(Address address, Object object, Object expectedObject, @ConstantParameter boolean doLoad, @ConstantParameter boolean nullCheck,
+ @ConstantParameter Register threadRegister, @ConstantParameter boolean trace) {
+ if (nullCheck) {
+ NullCheckNode.nullCheck(address);
+ }
+ Word thread = registerAsWord(threadRegister);
+ verifyOop(object);
+ Object fixedExpectedObject = FixedValueAnchorNode.getObject(expectedObject);
+ Pointer field = Word.fromAddress(address);
+ Pointer previousOop = Word.objectToTrackedPointer(fixedExpectedObject);
+ byte markingValue = thread.readByte(g1SATBQueueMarkingOffset(INJECTED_VMCONFIG));
+ int gcCycle = 0;
+ if (trace) {
+ gcCycle = (int) Word.unsigned(HotSpotReplacementsUtil.gcTotalCollectionsAddress(INJECTED_VMCONFIG)).readLong(0);
+ log(trace, "[%d] G1-Pre Thread %p Object %p\n", gcCycle, thread.rawValue(), Word.objectToTrackedPointer(object).rawValue());
+ log(trace, "[%d] G1-Pre Thread %p Expected Object %p\n", gcCycle, thread.rawValue(), Word.objectToTrackedPointer(fixedExpectedObject).rawValue());
+ log(trace, "[%d] G1-Pre Thread %p Field %p\n", gcCycle, thread.rawValue(), field.rawValue());
+ log(trace, "[%d] G1-Pre Thread %p Marking %d\n", gcCycle, thread.rawValue(), markingValue);
+ log(trace, "[%d] G1-Pre Thread %p DoLoad %d\n", gcCycle, thread.rawValue(), doLoad ? 1L : 0L);
+ }
+ g1AttemptedPreWriteBarrierCounter.inc();
+ // If the concurrent marker is enabled, the barrier is issued.
+ if (probability(NOT_FREQUENT_PROBABILITY, markingValue != (byte) 0)) {
+ // If the previous value has to be loaded (before the write), the load is issued.
+ // The load is always issued except the cases of CAS and referent field.
+ if (probability(LIKELY_PROBABILITY, doLoad)) {
+ previousOop = Word.objectToTrackedPointer(field.readObject(0, BarrierType.NONE));
+ if (trace) {
+ log(trace, "[%d] G1-Pre Thread %p Previous Object %p\n ", gcCycle, thread.rawValue(), previousOop.rawValue());
+ verifyOop(previousOop.toObject());
+ }
+ }
+ g1EffectivePreWriteBarrierCounter.inc();
+ // If the previous value is null the barrier should not be issued.
+ if (probability(FREQUENT_PROBABILITY, previousOop.notEqual(0))) {
+ g1ExecutedPreWriteBarrierCounter.inc();
+ // If the thread-local SATB buffer is full issue a native call which will
+ // initialize a new one and add the entry.
+ Word indexAddress = thread.add(g1SATBQueueIndexOffset(INJECTED_VMCONFIG));
+ Word indexValue = indexAddress.readWord(0);
+ if (probability(FREQUENT_PROBABILITY, indexValue.notEqual(0))) {
+ Word bufferAddress = thread.readWord(g1SATBQueueBufferOffset(INJECTED_VMCONFIG));
+ Word nextIndex = indexValue.subtract(wordSize());
+ Word logAddress = bufferAddress.add(nextIndex);
+ // Log the object to be marked as well as update the SATB's buffer next index.
+ logAddress.writeWord(0, previousOop, GC_LOG_LOCATION);
+ indexAddress.writeWord(0, nextIndex, GC_INDEX_LOCATION);
+ } else {
+ g1PreBarrierStub(G1WBPRECALL, previousOop.toObject());
+ }
+ }
+ }
+ }
+
+ @Snippet
+ public static void g1PostWriteBarrier(Address address, Object object, Object value, @ConstantParameter boolean usePrecise, @ConstantParameter Register threadRegister,
+ @ConstantParameter boolean trace) {
+ Word thread = registerAsWord(threadRegister);
+ Object fixedValue = FixedValueAnchorNode.getObject(value);
+ verifyOop(object);
+ verifyOop(fixedValue);
+ validateObject(object, fixedValue);
+ Pointer oop;
+ if (usePrecise) {
+ oop = Word.fromAddress(address);
+ } else {
+ oop = Word.objectToTrackedPointer(object);
+ }
+ int gcCycle = 0;
+ if (trace) {
+ gcCycle = (int) Word.unsigned(HotSpotReplacementsUtil.gcTotalCollectionsAddress(INJECTED_VMCONFIG)).readLong(0);
+ log(trace, "[%d] G1-Post Thread: %p Object: %p\n", gcCycle, thread.rawValue(), Word.objectToTrackedPointer(object).rawValue());
+ log(trace, "[%d] G1-Post Thread: %p Field: %p\n", gcCycle, thread.rawValue(), oop.rawValue());
+ }
+ Pointer writtenValue = Word.objectToTrackedPointer(fixedValue);
+ // The result of the xor reveals whether the installed pointer crosses heap regions.
+ // In case it does the write barrier has to be issued.
+ final int logOfHeapRegionGrainBytes = GraalHotSpotVMConfigNode.logOfHeapRegionGrainBytes();
+ Unsigned xorResult = (oop.xor(writtenValue)).unsignedShiftRight(logOfHeapRegionGrainBytes);
+
+ // Calculate the address of the card to be enqueued to the
+ // thread local card queue.
+ Unsigned cardBase = oop.unsignedShiftRight(cardTableShift(INJECTED_VMCONFIG));
+ final long startAddress = GraalHotSpotVMConfigNode.cardTableAddress();
+ int displacement = 0;
+ if (((int) startAddress) == startAddress && GraalHotSpotVMConfigNode.isCardTableAddressConstant()) {
+ displacement = (int) startAddress;
+ } else {
+ cardBase = cardBase.add(Word.unsigned(startAddress));
+ }
+ Word cardAddress = (Word) cardBase.add(displacement);
+
+ g1AttemptedPostWriteBarrierCounter.inc();
+ if (probability(FREQUENT_PROBABILITY, xorResult.notEqual(0))) {
+ g1EffectiveAfterXORPostWriteBarrierCounter.inc();
+
+ // If the written value is not null continue with the barrier addition.
+ if (probability(FREQUENT_PROBABILITY, writtenValue.notEqual(0))) {
+ byte cardByte = cardAddress.readByte(0, GC_CARD_LOCATION);
+ g1EffectiveAfterNullPostWriteBarrierCounter.inc();
+
+ // If the card is already dirty, (hence already enqueued) skip the insertion.
+ if (probability(NOT_FREQUENT_PROBABILITY, cardByte != g1YoungCardValue(INJECTED_VMCONFIG))) {
+ MembarNode.memoryBarrier(STORE_LOAD, GC_CARD_LOCATION);
+ byte cardByteReload = cardAddress.readByte(0, GC_CARD_LOCATION);
+ if (probability(NOT_FREQUENT_PROBABILITY, cardByteReload != dirtyCardValue(INJECTED_VMCONFIG))) {
+ log(trace, "[%d] G1-Post Thread: %p Card: %p \n", gcCycle, thread.rawValue(), Word.unsigned(cardByte).rawValue());
+ cardAddress.writeByte(0, (byte) 0, GC_CARD_LOCATION);
+ g1ExecutedPostWriteBarrierCounter.inc();
+
+ // If the thread local card queue is full, issue a native call which will
+ // initialize a new one and add the card entry.
+ Word indexAddress = thread.add(g1CardQueueIndexOffset(INJECTED_VMCONFIG));
+ Word indexValue = thread.readWord(g1CardQueueIndexOffset(INJECTED_VMCONFIG));
+ if (probability(FREQUENT_PROBABILITY, indexValue.notEqual(0))) {
+ Word bufferAddress = thread.readWord(g1CardQueueBufferOffset(INJECTED_VMCONFIG));
+ Word nextIndex = indexValue.subtract(wordSize());
+ Word logAddress = bufferAddress.add(nextIndex);
+ // Log the object to be scanned as well as update
+ // the card queue's next index.
+ logAddress.writeWord(0, cardAddress, GC_LOG_LOCATION);
+ indexAddress.writeWord(0, nextIndex, GC_INDEX_LOCATION);
+ } else {
+ g1PostBarrierStub(G1WBPOSTCALL, cardAddress);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ @Snippet
+ public static void g1ArrayRangePreWriteBarrier(Object object, int startIndex, int length, @ConstantParameter Register threadRegister) {
+ Word thread = registerAsWord(threadRegister);
+ byte markingValue = thread.readByte(g1SATBQueueMarkingOffset(INJECTED_VMCONFIG));
+ // If the concurrent marker is not enabled or the vector length is zero, return.
+ if (markingValue == (byte) 0 || length == 0) {
+ return;
+ }
+ Object dest = FixedValueAnchorNode.getObject(object);
+ Word bufferAddress = thread.readWord(g1SATBQueueBufferOffset(INJECTED_VMCONFIG));
+ Word indexAddress = thread.add(g1SATBQueueIndexOffset(INJECTED_VMCONFIG));
+ long dstAddr = GetObjectAddressNode.get(dest);
+ long indexValue = indexAddress.readWord(0).rawValue();
+ final int scale = arrayIndexScale(JavaKind.Object);
+ int header = arrayBaseOffset(JavaKind.Object);
+
+ for (int i = startIndex; i < length; i++) {
+ long address = dstAddr + header + (i * scale);
+ Pointer oop = Word.objectToTrackedPointer(Word.unsigned(address).readObject(0, BarrierType.NONE));
+ verifyOop(oop.toObject());
+ if (oop.notEqual(0)) {
+ if (indexValue != 0) {
+ indexValue = indexValue - wordSize();
+ Word logAddress = bufferAddress.add(Word.unsigned(indexValue));
+ // Log the object to be marked as well as update the SATB's buffer next index.
+ logAddress.writeWord(0, oop, GC_LOG_LOCATION);
+ indexAddress.writeWord(0, Word.unsigned(indexValue), GC_INDEX_LOCATION);
+ } else {
+ g1PreBarrierStub(G1WBPRECALL, oop.toObject());
+ }
+ }
+ }
+ }
+
+ @Snippet
+ public static void g1ArrayRangePostWriteBarrier(Object object, int startIndex, int length, @ConstantParameter Register threadRegister) {
+ if (length == 0) {
+ return;
+ }
+ Object dest = FixedValueAnchorNode.getObject(object);
+ Word thread = registerAsWord(threadRegister);
+ Word bufferAddress = thread.readWord(g1CardQueueBufferOffset(INJECTED_VMCONFIG));
+ Word indexAddress = thread.add(g1CardQueueIndexOffset(INJECTED_VMCONFIG));
+ long indexValue = thread.readWord(g1CardQueueIndexOffset(INJECTED_VMCONFIG)).rawValue();
+
+ int cardShift = cardTableShift(INJECTED_VMCONFIG);
+ final long cardStart = GraalHotSpotVMConfigNode.cardTableAddress();
+ final int scale = arrayIndexScale(JavaKind.Object);
+ int header = arrayBaseOffset(JavaKind.Object);
+ long dstAddr = GetObjectAddressNode.get(dest);
+ long start = (dstAddr + header + (long) startIndex * scale) >>> cardShift;
+ long end = (dstAddr + header + ((long) startIndex + length - 1) * scale) >>> cardShift;
+ long count = end - start + 1;
+
+ while (count-- > 0) {
+ Word cardAddress = Word.unsigned((start + cardStart) + count);
+ byte cardByte = cardAddress.readByte(0, GC_CARD_LOCATION);
+ // If the card is already dirty, (hence already enqueued) skip the insertion.
+ if (probability(NOT_FREQUENT_PROBABILITY, cardByte != g1YoungCardValue(INJECTED_VMCONFIG))) {
+ MembarNode.memoryBarrier(STORE_LOAD, GC_CARD_LOCATION);
+ byte cardByteReload = cardAddress.readByte(0, GC_CARD_LOCATION);
+ if (probability(NOT_FREQUENT_PROBABILITY, cardByteReload != dirtyCardValue(INJECTED_VMCONFIG))) {
+ cardAddress.writeByte(0, (byte) 0, GC_CARD_LOCATION);
+ // If the thread local card queue is full, issue a native call which will
+ // initialize a new one and add the card entry.
+ if (indexValue != 0) {
+ indexValue = indexValue - wordSize();
+ Word logAddress = bufferAddress.add(Word.unsigned(indexValue));
+ // Log the object to be scanned as well as update
+ // the card queue's next index.
+ logAddress.writeWord(0, cardAddress, GC_LOG_LOCATION);
+ indexAddress.writeWord(0, Word.unsigned(indexValue), GC_INDEX_LOCATION);
+ } else {
+ g1PostBarrierStub(G1WBPOSTCALL, cardAddress);
+ }
+ }
+ }
+ }
+ }
+
+ public static final ForeignCallDescriptor G1WBPRECALL = new ForeignCallDescriptor("write_barrier_pre", void.class, Object.class);
+
+ @NodeIntrinsic(ForeignCallNode.class)
+ private static native void g1PreBarrierStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Object object);
+
+ public static final ForeignCallDescriptor G1WBPOSTCALL = new ForeignCallDescriptor("write_barrier_post", void.class, Word.class);
+
+ @NodeIntrinsic(ForeignCallNode.class)
+ public static native void g1PostBarrierStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word card);
+
+ public static class Templates extends AbstractTemplates {
+
+ private final SnippetInfo serialImpreciseWriteBarrier = snippet(WriteBarrierSnippets.class, "serialImpreciseWriteBarrier", GC_CARD_LOCATION);
+ private final SnippetInfo serialPreciseWriteBarrier = snippet(WriteBarrierSnippets.class, "serialPreciseWriteBarrier", GC_CARD_LOCATION);
+ private final SnippetInfo serialArrayRangeWriteBarrier = snippet(WriteBarrierSnippets.class, "serialArrayRangeWriteBarrier");
+ private final SnippetInfo g1PreWriteBarrier = snippet(WriteBarrierSnippets.class, "g1PreWriteBarrier", GC_INDEX_LOCATION, GC_LOG_LOCATION);
+ private final SnippetInfo g1ReferentReadBarrier = snippet(WriteBarrierSnippets.class, "g1PreWriteBarrier", GC_INDEX_LOCATION, GC_LOG_LOCATION);
+ private final SnippetInfo g1PostWriteBarrier = snippet(WriteBarrierSnippets.class, "g1PostWriteBarrier", GC_CARD_LOCATION, GC_INDEX_LOCATION, GC_LOG_LOCATION);
+ private final SnippetInfo g1ArrayRangePreWriteBarrier = snippet(WriteBarrierSnippets.class, "g1ArrayRangePreWriteBarrier", GC_INDEX_LOCATION, GC_LOG_LOCATION);
+ private final SnippetInfo g1ArrayRangePostWriteBarrier = snippet(WriteBarrierSnippets.class, "g1ArrayRangePostWriteBarrier", GC_CARD_LOCATION, GC_INDEX_LOCATION, GC_LOG_LOCATION);
+
+ private final CompressEncoding oopEncoding;
+
+ public Templates(HotSpotProviders providers, TargetDescription target, CompressEncoding oopEncoding) {
+ super(providers, providers.getSnippetReflection(), target);
+ this.oopEncoding = oopEncoding;
+ }
+
+ public void lower(SerialWriteBarrier writeBarrier, LoweringTool tool) {
+ Arguments args;
+ if (writeBarrier.usePrecise()) {
+ args = new Arguments(serialPreciseWriteBarrier, writeBarrier.graph().getGuardsStage(), tool.getLoweringStage());
+ args.add("address", writeBarrier.getAddress());
+ } else {
+ args = new Arguments(serialImpreciseWriteBarrier, writeBarrier.graph().getGuardsStage(), tool.getLoweringStage());
+ OffsetAddressNode address = (OffsetAddressNode) writeBarrier.getAddress();
+ args.add("object", address.getBase());
+ }
+ template(args).instantiate(providers.getMetaAccess(), writeBarrier, DEFAULT_REPLACER, args);
+ }
+
+ public void lower(SerialArrayRangeWriteBarrier arrayRangeWriteBarrier, LoweringTool tool) {
+ Arguments args = new Arguments(serialArrayRangeWriteBarrier, arrayRangeWriteBarrier.graph().getGuardsStage(), tool.getLoweringStage());
+ args.add("object", arrayRangeWriteBarrier.getObject());
+ args.add("startIndex", arrayRangeWriteBarrier.getStartIndex());
+ args.add("length", arrayRangeWriteBarrier.getLength());
+ template(args).instantiate(providers.getMetaAccess(), arrayRangeWriteBarrier, DEFAULT_REPLACER, args);
+ }
+
+ public void lower(G1PreWriteBarrier writeBarrierPre, HotSpotRegistersProvider registers, LoweringTool tool) {
+ Arguments args = new Arguments(g1PreWriteBarrier, writeBarrierPre.graph().getGuardsStage(), tool.getLoweringStage());
+ AddressNode address = writeBarrierPre.getAddress();
+ args.add("address", address);
+ if (address instanceof OffsetAddressNode) {
+ args.add("object", ((OffsetAddressNode) address).getBase());
+ } else {
+ args.add("object", null);
+ }
+
+ ValueNode expected = writeBarrierPre.getExpectedObject();
+ if (expected != null && expected.stamp() instanceof NarrowOopStamp) {
+ assert oopEncoding != null;
+ expected = CompressionNode.uncompress(expected, oopEncoding);
+ }
+ args.add("expectedObject", expected);
+
+ args.addConst("doLoad", writeBarrierPre.doLoad());
+ args.addConst("nullCheck", writeBarrierPre.getNullCheck());
+ args.addConst("threadRegister", registers.getThreadRegister());
+ args.addConst("trace", traceBarrier());
+ template(args).instantiate(providers.getMetaAccess(), writeBarrierPre, DEFAULT_REPLACER, args);
+ }
+
+ public void lower(G1ReferentFieldReadBarrier readBarrier, HotSpotRegistersProvider registers, LoweringTool tool) {
+ Arguments args = new Arguments(g1ReferentReadBarrier, readBarrier.graph().getGuardsStage(), tool.getLoweringStage());
+ AddressNode address = readBarrier.getAddress();
+ args.add("address", address);
+ if (address instanceof OffsetAddressNode) {
+ args.add("object", ((OffsetAddressNode) address).getBase());
+ } else {
+ args.add("object", null);
+ }
+
+ ValueNode expected = readBarrier.getExpectedObject();
+ if (expected != null && expected.stamp() instanceof NarrowOopStamp) {
+ assert oopEncoding != null;
+ expected = CompressionNode.uncompress(expected, oopEncoding);
+ }
+
+ args.add("expectedObject", expected);
+ args.addConst("doLoad", readBarrier.doLoad());
+ args.addConst("nullCheck", false);
+ args.addConst("threadRegister", registers.getThreadRegister());
+ args.addConst("trace", traceBarrier());
+ template(args).instantiate(providers.getMetaAccess(), readBarrier, DEFAULT_REPLACER, args);
+ }
+
+ public void lower(G1PostWriteBarrier writeBarrierPost, HotSpotRegistersProvider registers, LoweringTool tool) {
+ StructuredGraph graph = writeBarrierPost.graph();
+ if (writeBarrierPost.alwaysNull()) {
+ graph.removeFixed(writeBarrierPost);
+ return;
+ }
+ Arguments args = new Arguments(g1PostWriteBarrier, graph.getGuardsStage(), tool.getLoweringStage());
+ AddressNode address = writeBarrierPost.getAddress();
+ args.add("address", address);
+ if (address instanceof OffsetAddressNode) {
+ args.add("object", ((OffsetAddressNode) address).getBase());
+ } else {
+ assert writeBarrierPost.usePrecise() : "found imprecise barrier that's not an object access " + writeBarrierPost;
+ args.add("object", null);
+ }
+
+ ValueNode value = writeBarrierPost.getValue();
+ if (value.stamp() instanceof NarrowOopStamp) {
+ assert oopEncoding != null;
+ value = CompressionNode.uncompress(value, oopEncoding);
+ }
+ args.add("value", value);
+
+ args.addConst("usePrecise", writeBarrierPost.usePrecise());
+ args.addConst("threadRegister", registers.getThreadRegister());
+ args.addConst("trace", traceBarrier());
+ template(args).instantiate(providers.getMetaAccess(), writeBarrierPost, DEFAULT_REPLACER, args);
+ }
+
+ public void lower(G1ArrayRangePreWriteBarrier arrayRangeWriteBarrier, HotSpotRegistersProvider registers, LoweringTool tool) {
+ Arguments args = new Arguments(g1ArrayRangePreWriteBarrier, arrayRangeWriteBarrier.graph().getGuardsStage(), tool.getLoweringStage());
+ args.add("object", arrayRangeWriteBarrier.getObject());
+ args.add("startIndex", arrayRangeWriteBarrier.getStartIndex());
+ args.add("length", arrayRangeWriteBarrier.getLength());
+ args.addConst("threadRegister", registers.getThreadRegister());
+ template(args).instantiate(providers.getMetaAccess(), arrayRangeWriteBarrier, DEFAULT_REPLACER, args);
+ }
+
+ public void lower(G1ArrayRangePostWriteBarrier arrayRangeWriteBarrier, HotSpotRegistersProvider registers, LoweringTool tool) {
+ Arguments args = new Arguments(g1ArrayRangePostWriteBarrier, arrayRangeWriteBarrier.graph().getGuardsStage(), tool.getLoweringStage());
+ args.add("object", arrayRangeWriteBarrier.getObject());
+ args.add("startIndex", arrayRangeWriteBarrier.getStartIndex());
+ args.add("length", arrayRangeWriteBarrier.getLength());
+ args.addConst("threadRegister", registers.getThreadRegister());
+ template(args).instantiate(providers.getMetaAccess(), arrayRangeWriteBarrier, DEFAULT_REPLACER, args);
+ }
+ }
+
+ /**
+ * Log method of debugging purposes.
+ */
+ public static void log(boolean enabled, String format, long value) {
+ if (enabled) {
+ Log.printf(format, value);
+ }
+ }
+
+ public static void log(boolean enabled, String format, long value1, long value2) {
+ if (enabled) {
+ Log.printf(format, value1, value2);
+ }
+ }
+
+ public static void log(boolean enabled, String format, long value1, long value2, long value3) {
+ if (enabled) {
+ Log.printf(format, value1, value2, value3);
+ }
+ }
+
+ public static boolean traceBarrier() {
+ return GraalOptions.GCDebugStartCycle.getValue() > 0 &&
+ ((int) Word.unsigned(HotSpotReplacementsUtil.gcTotalCollectionsAddress(INJECTED_VMCONFIG)).readLong(0) > GraalOptions.GCDebugStartCycle.getValue());
+ }
+
+ /**
+ * Validation helper method which performs sanity checks on write operations. The addresses of
+ * both the object and the value being written are checked in order to determine if they reside
+ * in a valid heap region. If an object is stale, an invalid access is performed in order to
+ * prematurely crash the VM and debug the stack trace of the faulty method.
+ */
+ public static void validateObject(Object parent, Object child) {
+ if (verifyOops(INJECTED_VMCONFIG) && child != null && !validateOop(VALIDATE_OBJECT, parent, child)) {
+ log(true, "Verification ERROR, Parent: %p Child: %p\n", Word.objectToTrackedPointer(parent).rawValue(), Word.objectToTrackedPointer(child).rawValue());
+ DirectObjectStoreNode.storeObject(null, 0, 0, null, LocationIdentity.any(), JavaKind.Object);
+ }
+ }
+
+ public static final ForeignCallDescriptor VALIDATE_OBJECT = new ForeignCallDescriptor("validate_object", boolean.class, Word.class, Word.class);
+
+ @NodeIntrinsic(ForeignCallNode.class)
+ private static native boolean validateOop(@ConstantNodeParameter ForeignCallDescriptor descriptor, Object parent, Object object);
+
+}