1 /*

     2  * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved.

     3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

     4  *

     5  * This code is free software; you can redistribute it and/or modify it

     6  * under the terms of the GNU General Public License version 2 only, as

     7  * published by the Free Software Foundation.

     8  *

     9  * This code is distributed in the hope that it will be useful, but WITHOUT

    10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    12  * version 2 for more details (a copy is included in the LICENSE file that

    13  * accompanied this code).

    14  *

    15  * You should have received a copy of the GNU General Public License version

    16  * 2 along with this work; if not, write to the Free Software Foundation,

    17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    18  *

    19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

    20  * or visit www.oracle.com if you need additional information or have any

    21  * questions.

    22  */

    23 

    24 

    25 package org.graalvm.compiler.hotspot.replacements;

    26 

    27 import static jdk.vm.ci.code.MemoryBarriers.STORE_LOAD;

    28 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;

    29 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_METAACCESS;

    30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.cardTableShift;

    31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.dirtyCardValue;

    32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1CardQueueBufferOffset;

    33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1CardQueueIndexOffset;

    34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1SATBQueueBufferOffset;

    35 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1SATBQueueIndexOffset;

    36 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1SATBQueueMarkingOffset;

    37 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1YoungCardValue;

    38 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;

    39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;

    40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOops;

    41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize;

    42 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;

    43 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.LIKELY_PROBABILITY;

    44 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY;

    45 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;

    46 import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;

    47 

    48 import org.graalvm.compiler.api.replacements.Snippet;

    49 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;

    50 import org.graalvm.compiler.core.common.CompressEncoding;

    51 import org.graalvm.compiler.core.common.GraalOptions;

    52 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;

    53 import org.graalvm.compiler.debug.DebugHandlersFactory;

    54 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;

    55 import org.graalvm.compiler.graph.Node.NodeIntrinsic;

    56 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;

    57 import org.graalvm.compiler.hotspot.gc.g1.G1ArrayRangePostWriteBarrier;

    58 import org.graalvm.compiler.hotspot.gc.g1.G1ArrayRangePreWriteBarrier;

    59 import org.graalvm.compiler.hotspot.gc.g1.G1PostWriteBarrier;

    60 import org.graalvm.compiler.hotspot.gc.g1.G1PreWriteBarrier;

    61 import org.graalvm.compiler.hotspot.gc.g1.G1ReferentFieldReadBarrier;

    62 import org.graalvm.compiler.hotspot.gc.shared.SerialArrayRangeWriteBarrier;

    63 import org.graalvm.compiler.hotspot.gc.shared.SerialWriteBarrier;

    64 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;

    65 import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;

    66 import org.graalvm.compiler.hotspot.nodes.GraalHotSpotVMConfigNode;

    67 import org.graalvm.compiler.hotspot.nodes.HotSpotCompressionNode;

    68 import org.graalvm.compiler.hotspot.nodes.VMErrorNode;

    69 import org.graalvm.compiler.nodes.NamedLocationIdentity;

    70 import org.graalvm.compiler.nodes.NodeView;

    71 import org.graalvm.compiler.nodes.PiNode;

    72 import org.graalvm.compiler.nodes.SnippetAnchorNode;

    73 import org.graalvm.compiler.nodes.StructuredGraph;

    74 import org.graalvm.compiler.nodes.ValueNode;

    75 import org.graalvm.compiler.nodes.extended.FixedValueAnchorNode;

    76 import org.graalvm.compiler.nodes.extended.ForeignCallNode;

    77 import org.graalvm.compiler.nodes.extended.MembarNode;

    78 import org.graalvm.compiler.nodes.extended.NullCheckNode;

    79 import org.graalvm.compiler.nodes.memory.HeapAccess.BarrierType;

    80 import org.graalvm.compiler.nodes.memory.address.AddressNode;

    81 import org.graalvm.compiler.nodes.memory.address.AddressNode.Address;

    82 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;

    83 import org.graalvm.compiler.nodes.spi.LoweringTool;

    84 import org.graalvm.compiler.nodes.type.NarrowOopStamp;

    85 import org.graalvm.compiler.options.OptionValues;

    86 import org.graalvm.compiler.replacements.Log;

    87 import org.graalvm.compiler.replacements.ReplacementsUtil;

    88 import org.graalvm.compiler.replacements.SnippetCounter;

    89 import org.graalvm.compiler.replacements.SnippetCounter.Group;

    90 import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;

    91 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;

    92 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;

    93 import org.graalvm.compiler.replacements.Snippets;

    94 import org.graalvm.compiler.replacements.nodes.AssertionNode;

    95 import org.graalvm.compiler.replacements.nodes.DirectStoreNode;

    96 import org.graalvm.compiler.word.Word;

    97 import jdk.internal.vm.compiler.word.LocationIdentity;

    98 import jdk.internal.vm.compiler.word.Pointer;

    99 import jdk.internal.vm.compiler.word.UnsignedWord;

   100 import jdk.internal.vm.compiler.word.WordFactory;

   101 

   102 import jdk.vm.ci.code.Register;

   103 import jdk.vm.ci.code.TargetDescription;

   104 import jdk.vm.ci.meta.JavaKind;

   105 

   106 public class WriteBarrierSnippets implements Snippets {

   107 

   108     static class Counters {

   109         Counters(SnippetCounter.Group.Factory factory) {

   110             Group countersWriteBarriers = factory.createSnippetCounterGroup("WriteBarriers");

   111             serialWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "serialWriteBarrier", "Number of Serial Write Barriers");

   112             g1AttemptedPreWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1AttemptedPreWriteBarrier", "Number of attempted G1 Pre Write Barriers");

   113             g1EffectivePreWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1EffectivePreWriteBarrier", "Number of effective G1 Pre Write Barriers");

   114             g1ExecutedPreWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1ExecutedPreWriteBarrier", "Number of executed G1 Pre Write Barriers");

   115             g1AttemptedPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1AttemptedPostWriteBarrier", "Number of attempted G1 Post Write Barriers");

   116             g1EffectiveAfterXORPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1EffectiveAfterXORPostWriteBarrier",

   117                             "Number of effective G1 Post Write Barriers (after passing the XOR test)");

   118             g1EffectiveAfterNullPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1EffectiveAfterNullPostWriteBarrier",

   119                             "Number of effective G1 Post Write Barriers (after passing the NULL test)");

   120             g1ExecutedPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1ExecutedPostWriteBarrier", "Number of executed G1 Post Write Barriers");

   121 

   122         }

   123 

   124         final SnippetCounter serialWriteBarrierCounter;

   125         final SnippetCounter g1AttemptedPreWriteBarrierCounter;

   126         final SnippetCounter g1EffectivePreWriteBarrierCounter;

   127         final SnippetCounter g1ExecutedPreWriteBarrierCounter;

   128         final SnippetCounter g1AttemptedPostWriteBarrierCounter;

   129         final SnippetCounter g1EffectiveAfterXORPostWriteBarrierCounter;

   130         final SnippetCounter g1EffectiveAfterNullPostWriteBarrierCounter;

   131         final SnippetCounter g1ExecutedPostWriteBarrierCounter;

   132     }

   133 

   134     public static final LocationIdentity GC_CARD_LOCATION = NamedLocationIdentity.mutable("GC-Card");

   135     public static final LocationIdentity GC_LOG_LOCATION = NamedLocationIdentity.mutable("GC-Log");

   136     public static final LocationIdentity GC_INDEX_LOCATION = NamedLocationIdentity.mutable("GC-Index");

   137 

   138     private static void serialWriteBarrier(Pointer ptr, Counters counters) {

   139         counters.serialWriteBarrierCounter.inc();

   140         final long startAddress = GraalHotSpotVMConfigNode.cardTableAddress();

   141         Word base = (Word) ptr.unsignedShiftRight(cardTableShift(INJECTED_VMCONFIG));

   142         if (((int) startAddress) == startAddress && GraalHotSpotVMConfigNode.isCardTableAddressConstant()) {

   143             base.writeByte((int) startAddress, (byte) 0, GC_CARD_LOCATION);

   144         } else {

   145             base.writeByte(WordFactory.unsigned(startAddress), (byte) 0, GC_CARD_LOCATION);

   146         }

   147     }

   148 

   149     @Snippet

   150     public static void serialImpreciseWriteBarrier(Object object, @ConstantParameter boolean verifyBarrier, @ConstantParameter Counters counters) {

   151         if (verifyBarrier) {

   152             verifyNotArray(object);

   153         }

   154         serialWriteBarrier(Word.objectToTrackedPointer(object), counters);

   155     }

   156 

   157     @Snippet

   158     public static void serialPreciseWriteBarrier(Address address, @ConstantParameter Counters counters) {

   159         serialWriteBarrier(Word.fromAddress(address), counters);

   160     }

   161 

   162     @Snippet

   163     public static void serialArrayRangeWriteBarrier(Address address, int length, @ConstantParameter int elementStride) {

   164         if (length == 0) {

   165             return;

   166         }

   167         int cardShift = cardTableShift(INJECTED_VMCONFIG);

   168         final long cardStart = GraalHotSpotVMConfigNode.cardTableAddress();

   169         long start = getPointerToFirstArrayElement(address, length, elementStride) >>> cardShift;

   170         long end = getPointerToLastArrayElement(address, length, elementStride) >>> cardShift;

   171         long count = end - start + 1;

   172         while (count-- > 0) {

   173             DirectStoreNode.storeBoolean((start + cardStart) + count, false, JavaKind.Boolean);

   174         }

   175     }

   176 

   177     @Snippet

   178     public static void g1PreWriteBarrier(Address address, Object object, Object expectedObject, @ConstantParameter boolean doLoad, @ConstantParameter boolean nullCheck,

   179                     @ConstantParameter Register threadRegister, @ConstantParameter boolean trace, @ConstantParameter Counters counters) {

   180         if (nullCheck) {

   181             NullCheckNode.nullCheck(address);

   182         }

   183         Word thread = registerAsWord(threadRegister);

   184         verifyOop(object);

   185         Object fixedExpectedObject = FixedValueAnchorNode.getObject(expectedObject);

   186         Word field = Word.fromAddress(address);

   187         Pointer previousOop = Word.objectToTrackedPointer(fixedExpectedObject);

   188         byte markingValue = thread.readByte(g1SATBQueueMarkingOffset(INJECTED_VMCONFIG));

   189         int gcCycle = 0;

   190         if (trace) {

   191             Pointer gcTotalCollectionsAddress = WordFactory.pointer(HotSpotReplacementsUtil.gcTotalCollectionsAddress(INJECTED_VMCONFIG));

   192             gcCycle = (int) gcTotalCollectionsAddress.readLong(0);

   193             log(trace, "[%d] G1-Pre Thread %p Object %p\n", gcCycle, thread.rawValue(), Word.objectToTrackedPointer(object).rawValue());

   194             log(trace, "[%d] G1-Pre Thread %p Expected Object %p\n", gcCycle, thread.rawValue(), Word.objectToTrackedPointer(fixedExpectedObject).rawValue());

   195             log(trace, "[%d] G1-Pre Thread %p Field %p\n", gcCycle, thread.rawValue(), field.rawValue());

   196             log(trace, "[%d] G1-Pre Thread %p Marking %d\n", gcCycle, thread.rawValue(), markingValue);

   197             log(trace, "[%d] G1-Pre Thread %p DoLoad %d\n", gcCycle, thread.rawValue(), doLoad ? 1L : 0L);

   198         }

   199         counters.g1AttemptedPreWriteBarrierCounter.inc();

   200         // If the concurrent marker is enabled, the barrier is issued.

   201         if (probability(NOT_FREQUENT_PROBABILITY, markingValue != (byte) 0)) {

   202             // If the previous value has to be loaded (before the write), the load is issued.

   203             // The load is always issued except the cases of CAS and referent field.

   204             if (probability(LIKELY_PROBABILITY, doLoad)) {

   205                 previousOop = Word.objectToTrackedPointer(field.readObject(0, BarrierType.NONE));

   206                 if (trace) {

   207                     log(trace, "[%d] G1-Pre Thread %p Previous Object %p\n ", gcCycle, thread.rawValue(), previousOop.rawValue());

   208                     verifyOop(previousOop.toObject());

   209                 }

   210             }

   211             counters.g1EffectivePreWriteBarrierCounter.inc();

   212             // If the previous value is null the barrier should not be issued.

   213             if (probability(FREQUENT_PROBABILITY, previousOop.notEqual(0))) {

   214                 counters.g1ExecutedPreWriteBarrierCounter.inc();

   215                 // If the thread-local SATB buffer is full issue a native call which will

   216                 // initialize a new one and add the entry.

   217                 Word indexAddress = thread.add(g1SATBQueueIndexOffset(INJECTED_VMCONFIG));

   218                 Word indexValue = indexAddress.readWord(0);

   219                 if (probability(FREQUENT_PROBABILITY, indexValue.notEqual(0))) {

   220                     Word bufferAddress = thread.readWord(g1SATBQueueBufferOffset(INJECTED_VMCONFIG));

   221                     Word nextIndex = indexValue.subtract(wordSize());

   222                     Word logAddress = bufferAddress.add(nextIndex);

   223                     // Log the object to be marked as well as update the SATB's buffer next index.

   224                     logAddress.writeWord(0, previousOop, GC_LOG_LOCATION);

   225                     indexAddress.writeWord(0, nextIndex, GC_INDEX_LOCATION);

   226                 } else {

   227                     g1PreBarrierStub(G1WBPRECALL, previousOop.toObject());

   228                 }

   229             }

   230         }

   231     }

   232 

   233     @Snippet

   234     public static void g1PostWriteBarrier(Address address, Object object, Object value, @ConstantParameter boolean usePrecise, @ConstantParameter boolean verifyBarrier,

   235                     @ConstantParameter Register threadRegister, @ConstantParameter boolean trace, @ConstantParameter Counters counters) {

   236         Word thread = registerAsWord(threadRegister);

   237         Object fixedValue = FixedValueAnchorNode.getObject(value);

   238         verifyOop(object);

   239         verifyOop(fixedValue);

   240         validateObject(object, fixedValue);

   241         Pointer oop;

   242         if (usePrecise) {

   243             oop = Word.fromAddress(address);

   244         } else {

   245             if (verifyBarrier) {

   246                 verifyNotArray(object);

   247             }

   248             oop = Word.objectToTrackedPointer(object);

   249         }

   250         int gcCycle = 0;

   251         if (trace) {

   252             Pointer gcTotalCollectionsAddress = WordFactory.pointer(HotSpotReplacementsUtil.gcTotalCollectionsAddress(INJECTED_VMCONFIG));

   253             gcCycle = (int) gcTotalCollectionsAddress.readLong(0);

   254             log(trace, "[%d] G1-Post Thread: %p Object: %p\n", gcCycle, thread.rawValue(), Word.objectToTrackedPointer(object).rawValue());

   255             log(trace, "[%d] G1-Post Thread: %p Field: %p\n", gcCycle, thread.rawValue(), oop.rawValue());

   256         }

   257         Pointer writtenValue = Word.objectToTrackedPointer(fixedValue);

   258         // The result of the xor reveals whether the installed pointer crosses heap regions.

   259         // In case it does the write barrier has to be issued.

   260         final int logOfHeapRegionGrainBytes = GraalHotSpotVMConfigNode.logOfHeapRegionGrainBytes();

   261         UnsignedWord xorResult = (oop.xor(writtenValue)).unsignedShiftRight(logOfHeapRegionGrainBytes);

   262 

   263         // Calculate the address of the card to be enqueued to the

   264         // thread local card queue.

   265         UnsignedWord cardBase = oop.unsignedShiftRight(cardTableShift(INJECTED_VMCONFIG));

   266         final long startAddress = GraalHotSpotVMConfigNode.cardTableAddress();

   267         int displacement = 0;

   268         if (((int) startAddress) == startAddress && GraalHotSpotVMConfigNode.isCardTableAddressConstant()) {

   269             displacement = (int) startAddress;

   270         } else {

   271             cardBase = cardBase.add(WordFactory.unsigned(startAddress));

   272         }

   273         Word cardAddress = (Word) cardBase.add(displacement);

   274 

   275         counters.g1AttemptedPostWriteBarrierCounter.inc();

   276         if (probability(FREQUENT_PROBABILITY, xorResult.notEqual(0))) {

   277             counters.g1EffectiveAfterXORPostWriteBarrierCounter.inc();

   278 

   279             // If the written value is not null continue with the barrier addition.

   280             if (probability(FREQUENT_PROBABILITY, writtenValue.notEqual(0))) {

   281                 byte cardByte = cardAddress.readByte(0, GC_CARD_LOCATION);

   282                 counters.g1EffectiveAfterNullPostWriteBarrierCounter.inc();

   283 

   284                 // If the card is already dirty, (hence already enqueued) skip the insertion.

   285                 if (probability(NOT_FREQUENT_PROBABILITY, cardByte != g1YoungCardValue(INJECTED_VMCONFIG))) {

   286                     MembarNode.memoryBarrier(STORE_LOAD, GC_CARD_LOCATION);

   287                     byte cardByteReload = cardAddress.readByte(0, GC_CARD_LOCATION);

   288                     if (probability(NOT_FREQUENT_PROBABILITY, cardByteReload != dirtyCardValue(INJECTED_VMCONFIG))) {

   289                         log(trace, "[%d] G1-Post Thread: %p Card: %p \n", gcCycle, thread.rawValue(), WordFactory.unsigned((int) cardByte).rawValue());

   290                         cardAddress.writeByte(0, (byte) 0, GC_CARD_LOCATION);

   291                         counters.g1ExecutedPostWriteBarrierCounter.inc();

   292 

   293                         // If the thread local card queue is full, issue a native call which will

   294                         // initialize a new one and add the card entry.

   295                         Word indexAddress = thread.add(g1CardQueueIndexOffset(INJECTED_VMCONFIG));

   296                         Word indexValue = thread.readWord(g1CardQueueIndexOffset(INJECTED_VMCONFIG));

   297                         if (probability(FREQUENT_PROBABILITY, indexValue.notEqual(0))) {

   298                             Word bufferAddress = thread.readWord(g1CardQueueBufferOffset(INJECTED_VMCONFIG));

   299                             Word nextIndex = indexValue.subtract(wordSize());

   300                             Word logAddress = bufferAddress.add(nextIndex);

   301                             // Log the object to be scanned as well as update

   302                             // the card queue's next index.

   303                             logAddress.writeWord(0, cardAddress, GC_LOG_LOCATION);

   304                             indexAddress.writeWord(0, nextIndex, GC_INDEX_LOCATION);

   305                         } else {

   306                             g1PostBarrierStub(G1WBPOSTCALL, cardAddress);

   307                         }

   308                     }

   309                 }

   310             }

   311         }

   312     }

   313 

   314     private static void verifyNotArray(Object object) {

   315         if (object != null) {

   316             // Manually build the null check and cast because we're in snippet that's lowered late.

   317             AssertionNode.assertion(false, !PiNode.piCastNonNull(object, SnippetAnchorNode.anchor()).getClass().isArray(), "imprecise card mark used with array");

   318         }

   319     }

   320 

   321     @Snippet

   322     public static void g1ArrayRangePreWriteBarrier(Address address, int length, @ConstantParameter int elementStride, @ConstantParameter Register threadRegister) {

   323         Word thread = registerAsWord(threadRegister);

   324         byte markingValue = thread.readByte(g1SATBQueueMarkingOffset(INJECTED_VMCONFIG));

   325         // If the concurrent marker is not enabled or the vector length is zero, return.

   326         if (markingValue == (byte) 0 || length == 0) {

   327             return;

   328         }

   329         Word bufferAddress = thread.readWord(g1SATBQueueBufferOffset(INJECTED_VMCONFIG));

   330         Word indexAddress = thread.add(g1SATBQueueIndexOffset(INJECTED_VMCONFIG));

   331         long indexValue = indexAddress.readWord(0).rawValue();

   332         final int scale = ReplacementsUtil.arrayIndexScale(INJECTED_METAACCESS, JavaKind.Object);

   333         long start = getPointerToFirstArrayElement(address, length, elementStride);

   334 

   335         for (int i = 0; i < length; i++) {

   336             Word arrElemPtr = WordFactory.pointer(start + i * scale);

   337             Pointer oop = Word.objectToTrackedPointer(arrElemPtr.readObject(0, BarrierType.NONE));

   338             verifyOop(oop.toObject());

   339             if (oop.notEqual(0)) {

   340                 if (indexValue != 0) {

   341                     indexValue = indexValue - wordSize();

   342                     Word logAddress = bufferAddress.add(WordFactory.unsigned(indexValue));

   343                     // Log the object to be marked as well as update the SATB's buffer next index.

   344                     logAddress.writeWord(0, oop, GC_LOG_LOCATION);

   345                     indexAddress.writeWord(0, WordFactory.unsigned(indexValue), GC_INDEX_LOCATION);

   346                 } else {

   347                     g1PreBarrierStub(G1WBPRECALL, oop.toObject());

   348                 }

   349             }

   350         }

   351     }

   352 

   353     @Snippet

   354     public static void g1ArrayRangePostWriteBarrier(Address address, int length, @ConstantParameter int elementStride, @ConstantParameter Register threadRegister) {

   355         if (length == 0) {

   356             return;

   357         }

   358         Word thread = registerAsWord(threadRegister);

   359         Word bufferAddress = thread.readWord(g1CardQueueBufferOffset(INJECTED_VMCONFIG));

   360         Word indexAddress = thread.add(g1CardQueueIndexOffset(INJECTED_VMCONFIG));

   361         long indexValue = thread.readWord(g1CardQueueIndexOffset(INJECTED_VMCONFIG)).rawValue();

   362 

   363         int cardShift = cardTableShift(INJECTED_VMCONFIG);

   364         final long cardStart = GraalHotSpotVMConfigNode.cardTableAddress();

   365         long start = getPointerToFirstArrayElement(address, length, elementStride) >>> cardShift;

   366         long end = getPointerToLastArrayElement(address, length, elementStride) >>> cardShift;

   367         long count = end - start + 1;

   368 

   369         while (count-- > 0) {

   370             Word cardAddress = WordFactory.unsigned((start + cardStart) + count);

   371             byte cardByte = cardAddress.readByte(0, GC_CARD_LOCATION);

   372             // If the card is already dirty, (hence already enqueued) skip the insertion.

   373             if (probability(NOT_FREQUENT_PROBABILITY, cardByte != g1YoungCardValue(INJECTED_VMCONFIG))) {

   374                 MembarNode.memoryBarrier(STORE_LOAD, GC_CARD_LOCATION);

   375                 byte cardByteReload = cardAddress.readByte(0, GC_CARD_LOCATION);

   376                 if (probability(NOT_FREQUENT_PROBABILITY, cardByteReload != dirtyCardValue(INJECTED_VMCONFIG))) {

   377                     cardAddress.writeByte(0, (byte) 0, GC_CARD_LOCATION);

   378                     // If the thread local card queue is full, issue a native call which will

   379                     // initialize a new one and add the card entry.

   380                     if (indexValue != 0) {

   381                         indexValue = indexValue - wordSize();

   382                         Word logAddress = bufferAddress.add(WordFactory.unsigned(indexValue));

   383                         // Log the object to be scanned as well as update

   384                         // the card queue's next index.

   385                         logAddress.writeWord(0, cardAddress, GC_LOG_LOCATION);

   386                         indexAddress.writeWord(0, WordFactory.unsigned(indexValue), GC_INDEX_LOCATION);

   387                     } else {

   388                         g1PostBarrierStub(G1WBPOSTCALL, cardAddress);

   389                     }

   390                 }

   391             }

   392         }

   393     }

   394 

   395     private static long getPointerToFirstArrayElement(Address address, int length, int elementStride) {

   396         long result = Word.fromAddress(address).rawValue();

   397         if (elementStride < 0) {

   398             // the address points to the place after the last array element

   399             result = result + elementStride * length;

   400         }

   401         return result;

   402     }

   403 

   404     private static long getPointerToLastArrayElement(Address address, int length, int elementStride) {

   405         long result = Word.fromAddress(address).rawValue();

   406         if (elementStride < 0) {

   407             // the address points to the place after the last array element

   408             result = result + elementStride;

   409         } else {

   410             result = result + (length - 1) * elementStride;

   411         }

   412         return result;

   413     }

   414 

   415     public static final ForeignCallDescriptor G1WBPRECALL = new ForeignCallDescriptor("write_barrier_pre", void.class, Object.class);

   416 

   417     @NodeIntrinsic(ForeignCallNode.class)

   418     private static native void g1PreBarrierStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Object object);

   419 

   420     public static final ForeignCallDescriptor G1WBPOSTCALL = new ForeignCallDescriptor("write_barrier_post", void.class, Word.class);

   421 

   422     @NodeIntrinsic(ForeignCallNode.class)

   423     public static native void g1PostBarrierStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word card);

   424 

   425     public static class Templates extends AbstractTemplates {

   426 

   427         private final SnippetInfo serialImpreciseWriteBarrier = snippet(WriteBarrierSnippets.class, "serialImpreciseWriteBarrier", GC_CARD_LOCATION);

   428         private final SnippetInfo serialPreciseWriteBarrier = snippet(WriteBarrierSnippets.class, "serialPreciseWriteBarrier", GC_CARD_LOCATION);

   429         private final SnippetInfo serialArrayRangeWriteBarrier = snippet(WriteBarrierSnippets.class, "serialArrayRangeWriteBarrier");

   430         private final SnippetInfo g1PreWriteBarrier = snippet(WriteBarrierSnippets.class, "g1PreWriteBarrier", GC_INDEX_LOCATION, GC_LOG_LOCATION);

   431         private final SnippetInfo g1ReferentReadBarrier = g1PreWriteBarrier;

   432         private final SnippetInfo g1PostWriteBarrier = snippet(WriteBarrierSnippets.class, "g1PostWriteBarrier", GC_CARD_LOCATION, GC_INDEX_LOCATION, GC_LOG_LOCATION);

   433         private final SnippetInfo g1ArrayRangePreWriteBarrier = snippet(WriteBarrierSnippets.class, "g1ArrayRangePreWriteBarrier", GC_INDEX_LOCATION, GC_LOG_LOCATION);

   434         private final SnippetInfo g1ArrayRangePostWriteBarrier = snippet(WriteBarrierSnippets.class, "g1ArrayRangePostWriteBarrier", GC_CARD_LOCATION, GC_INDEX_LOCATION, GC_LOG_LOCATION);

   435 

   436         private final CompressEncoding oopEncoding;

   437         private final Counters counters;

   438         private final boolean verifyBarrier;

   439         private final long gcTotalCollectionsAddress;

   440 

   441         public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, Group.Factory factory, HotSpotProviders providers, TargetDescription target,

   442                         GraalHotSpotVMConfig config) {

   443             super(options, factories, providers, providers.getSnippetReflection(), target);

   444             this.oopEncoding = config.useCompressedOops ? config.getOopEncoding() : null;

   445             this.verifyBarrier = ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED || config.verifyBeforeGC || config.verifyAfterGC;

   446             this.gcTotalCollectionsAddress = config.gcTotalCollectionsAddress();

   447             this.counters = new Counters(factory);

   448         }

   449 

   450         public boolean traceBarrier(StructuredGraph graph) {

   451             long startCycle = GraalOptions.GCDebugStartCycle.getValue(graph.getOptions());

   452             return startCycle > 0 && ((Pointer) WordFactory.pointer(gcTotalCollectionsAddress)).readLong(0) > startCycle;

   453         }

   454 

   455         public void lower(SerialWriteBarrier writeBarrier, LoweringTool tool) {

   456             Arguments args;

   457             if (writeBarrier.usePrecise()) {

   458                 args = new Arguments(serialPreciseWriteBarrier, writeBarrier.graph().getGuardsStage(), tool.getLoweringStage());

   459                 args.add("address", writeBarrier.getAddress());

   460             } else {

   461                 args = new Arguments(serialImpreciseWriteBarrier, writeBarrier.graph().getGuardsStage(), tool.getLoweringStage());

   462                 OffsetAddressNode address = (OffsetAddressNode) writeBarrier.getAddress();

   463                 args.add("object", address.getBase());

   464                 args.addConst("verifyBarrier", verifyBarrier);

   465             }

   466             args.addConst("counters", counters);

   467             template(writeBarrier, args).instantiate(providers.getMetaAccess(), writeBarrier, DEFAULT_REPLACER, args);

   468         }

   469 

   470         public void lower(SerialArrayRangeWriteBarrier arrayRangeWriteBarrier, LoweringTool tool) {

   471             Arguments args = new Arguments(serialArrayRangeWriteBarrier, arrayRangeWriteBarrier.graph().getGuardsStage(), tool.getLoweringStage());

   472             args.add("address", arrayRangeWriteBarrier.getAddress());

   473             args.add("length", arrayRangeWriteBarrier.getLength());

   474             args.addConst("elementStride", arrayRangeWriteBarrier.getElementStride());

   475             template(arrayRangeWriteBarrier, args).instantiate(providers.getMetaAccess(), arrayRangeWriteBarrier, DEFAULT_REPLACER, args);

   476         }

   477 

   478         public void lower(G1PreWriteBarrier writeBarrierPre, HotSpotRegistersProvider registers, LoweringTool tool) {

   479             Arguments args = new Arguments(g1PreWriteBarrier, writeBarrierPre.graph().getGuardsStage(), tool.getLoweringStage());

   480             AddressNode address = writeBarrierPre.getAddress();

   481             args.add("address", address);

   482             if (address instanceof OffsetAddressNode) {

   483                 args.add("object", ((OffsetAddressNode) address).getBase());

   484             } else {

   485                 args.add("object", null);

   486             }

   487 

   488             ValueNode expected = writeBarrierPre.getExpectedObject();

   489             if (expected != null && expected.stamp(NodeView.DEFAULT) instanceof NarrowOopStamp) {

   490                 assert oopEncoding != null;

   491                 expected = HotSpotCompressionNode.uncompress(expected, oopEncoding);

   492             }

   493             args.add("expectedObject", expected);

   494 

   495             args.addConst("doLoad", writeBarrierPre.doLoad());

   496             args.addConst("nullCheck", writeBarrierPre.getNullCheck());

   497             args.addConst("threadRegister", registers.getThreadRegister());

   498             args.addConst("trace", traceBarrier(writeBarrierPre.graph()));

   499             args.addConst("counters", counters);

   500             template(writeBarrierPre, args).instantiate(providers.getMetaAccess(), writeBarrierPre, DEFAULT_REPLACER, args);

   501         }

   502 

   503         public void lower(G1ReferentFieldReadBarrier readBarrier, HotSpotRegistersProvider registers, LoweringTool tool) {

   504             Arguments args = new Arguments(g1ReferentReadBarrier, readBarrier.graph().getGuardsStage(), tool.getLoweringStage());

   505             AddressNode address = readBarrier.getAddress();

   506             args.add("address", address);

   507             if (address instanceof OffsetAddressNode) {

   508                 args.add("object", ((OffsetAddressNode) address).getBase());

   509             } else {

   510                 args.add("object", null);

   511             }

   512 

   513             ValueNode expected = readBarrier.getExpectedObject();

   514             if (expected != null && expected.stamp(NodeView.DEFAULT) instanceof NarrowOopStamp) {

   515                 assert oopEncoding != null;

   516                 expected = HotSpotCompressionNode.uncompress(expected, oopEncoding);

   517             }

   518 

   519             args.add("expectedObject", expected);

   520             args.addConst("doLoad", readBarrier.doLoad());

   521             args.addConst("nullCheck", false);

   522             args.addConst("threadRegister", registers.getThreadRegister());

   523             args.addConst("trace", traceBarrier(readBarrier.graph()));

   524             args.addConst("counters", counters);

   525             template(readBarrier, args).instantiate(providers.getMetaAccess(), readBarrier, DEFAULT_REPLACER, args);

   526         }

   527 

   528         public void lower(G1PostWriteBarrier writeBarrierPost, HotSpotRegistersProvider registers, LoweringTool tool) {

   529             StructuredGraph graph = writeBarrierPost.graph();

   530             if (writeBarrierPost.alwaysNull()) {

   531                 graph.removeFixed(writeBarrierPost);

   532                 return;

   533             }

   534             Arguments args = new Arguments(g1PostWriteBarrier, graph.getGuardsStage(), tool.getLoweringStage());

   535             AddressNode address = writeBarrierPost.getAddress();

   536             args.add("address", address);

   537             if (address instanceof OffsetAddressNode) {

   538                 args.add("object", ((OffsetAddressNode) address).getBase());

   539             } else {

   540                 assert writeBarrierPost.usePrecise() : "found imprecise barrier that's not an object access " + writeBarrierPost;

   541                 args.add("object", null);

   542             }

   543 

   544             ValueNode value = writeBarrierPost.getValue();

   545             if (value.stamp(NodeView.DEFAULT) instanceof NarrowOopStamp) {

   546                 assert oopEncoding != null;

   547                 value = HotSpotCompressionNode.uncompress(value, oopEncoding);

   548             }

   549             args.add("value", value);

   550 

   551             args.addConst("usePrecise", writeBarrierPost.usePrecise());

   552             args.addConst("verifyBarrier", verifyBarrier);

   553             args.addConst("threadRegister", registers.getThreadRegister());

   554             args.addConst("trace", traceBarrier(writeBarrierPost.graph()));

   555             args.addConst("counters", counters);

   556             template(writeBarrierPost, args).instantiate(providers.getMetaAccess(), writeBarrierPost, DEFAULT_REPLACER, args);

   557         }

   558 

   559         public void lower(G1ArrayRangePreWriteBarrier arrayRangeWriteBarrier, HotSpotRegistersProvider registers, LoweringTool tool) {

   560             Arguments args = new Arguments(g1ArrayRangePreWriteBarrier, arrayRangeWriteBarrier.graph().getGuardsStage(), tool.getLoweringStage());

   561             args.add("address", arrayRangeWriteBarrier.getAddress());

   562             args.add("length", arrayRangeWriteBarrier.getLength());

   563             args.addConst("elementStride", arrayRangeWriteBarrier.getElementStride());

   564             args.addConst("threadRegister", registers.getThreadRegister());

   565             template(arrayRangeWriteBarrier, args).instantiate(providers.getMetaAccess(), arrayRangeWriteBarrier, DEFAULT_REPLACER, args);

   566         }

   567 

   568         public void lower(G1ArrayRangePostWriteBarrier arrayRangeWriteBarrier, HotSpotRegistersProvider registers, LoweringTool tool) {

   569             Arguments args = new Arguments(g1ArrayRangePostWriteBarrier, arrayRangeWriteBarrier.graph().getGuardsStage(), tool.getLoweringStage());

   570             args.add("address", arrayRangeWriteBarrier.getAddress());

   571             args.add("length", arrayRangeWriteBarrier.getLength());

   572             args.addConst("elementStride", arrayRangeWriteBarrier.getElementStride());

   573             args.addConst("threadRegister", registers.getThreadRegister());

   574             template(arrayRangeWriteBarrier, args).instantiate(providers.getMetaAccess(), arrayRangeWriteBarrier, DEFAULT_REPLACER, args);

   575         }

   576     }

   577 

   578     /**

   579      * Log method of debugging purposes.

   580      */

   581     public static void log(boolean enabled, String format, long value) {

   582         if (enabled) {

   583             Log.printf(format, value);

   584         }

   585     }

   586 

   587     public static void log(boolean enabled, String format, long value1, long value2) {

   588         if (enabled) {

   589             Log.printf(format, value1, value2);

   590         }

   591     }

   592 

   593     public static void log(boolean enabled, String format, long value1, long value2, long value3) {

   594         if (enabled) {

   595             Log.printf(format, value1, value2, value3);

   596         }

   597     }

   598 

   599     /**

   600      * Validation helper method which performs sanity checks on write operations. The addresses of

   601      * both the object and the value being written are checked in order to determine if they reside

   602      * in a valid heap region. If an object is stale, an invalid access is performed in order to

   603      * prematurely crash the VM and debug the stack trace of the faulty method.

   604      */

   605     public static void validateObject(Object parent, Object child) {

   606         if (verifyOops(INJECTED_VMCONFIG) && child != null) {

   607             Word parentWord = Word.objectToTrackedPointer(parent);

   608             Word childWord = Word.objectToTrackedPointer(child);

   609             if (!validateOop(VALIDATE_OBJECT, parentWord, childWord)) {

   610                 log(true, "Verification ERROR, Parent: %p Child: %p\n", parentWord.rawValue(), childWord.rawValue());

   611                 VMErrorNode.vmError("Verification ERROR, Parent: %p\n", parentWord.rawValue());

   612             }

   613         }

   614     }

   615 

   616     public static final ForeignCallDescriptor VALIDATE_OBJECT = new ForeignCallDescriptor("validate_object", boolean.class, Word.class, Word.class);

   617 

   618     @NodeIntrinsic(ForeignCallNode.class)

   619     private static native boolean validateOop(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word parent, Word object);

   620 

   621 }