45 |
45 |
46 import jdk.vm.ci.meta.JavaConstant; |
46 import jdk.vm.ci.meta.JavaConstant; |
47 import org.graalvm.compiler.api.directives.GraalDirectives; |
47 import org.graalvm.compiler.api.directives.GraalDirectives; |
48 import org.graalvm.compiler.api.replacements.Snippet; |
48 import org.graalvm.compiler.api.replacements.Snippet; |
49 import org.graalvm.compiler.api.replacements.SnippetReflectionProvider; |
49 import org.graalvm.compiler.api.replacements.SnippetReflectionProvider; |
|
50 import org.graalvm.compiler.core.common.GraalOptions; |
50 import org.graalvm.compiler.core.common.LIRKind; |
51 import org.graalvm.compiler.core.common.LIRKind; |
51 import org.graalvm.compiler.core.common.spi.ForeignCallsProvider; |
52 import org.graalvm.compiler.core.common.spi.ForeignCallsProvider; |
52 import org.graalvm.compiler.core.common.type.AbstractPointerStamp; |
53 import org.graalvm.compiler.core.common.type.AbstractPointerStamp; |
53 import org.graalvm.compiler.core.common.type.IntegerStamp; |
54 import org.graalvm.compiler.core.common.type.IntegerStamp; |
54 import org.graalvm.compiler.core.common.type.ObjectStamp; |
55 import org.graalvm.compiler.core.common.type.ObjectStamp; |
127 import org.graalvm.compiler.nodes.java.UnsafeCompareAndExchangeNode; |
128 import org.graalvm.compiler.nodes.java.UnsafeCompareAndExchangeNode; |
128 import org.graalvm.compiler.nodes.java.UnsafeCompareAndSwapNode; |
129 import org.graalvm.compiler.nodes.java.UnsafeCompareAndSwapNode; |
129 import org.graalvm.compiler.nodes.java.ValueCompareAndSwapNode; |
130 import org.graalvm.compiler.nodes.java.ValueCompareAndSwapNode; |
130 import org.graalvm.compiler.nodes.memory.HeapAccess.BarrierType; |
131 import org.graalvm.compiler.nodes.memory.HeapAccess.BarrierType; |
131 import org.graalvm.compiler.nodes.memory.ReadNode; |
132 import org.graalvm.compiler.nodes.memory.ReadNode; |
|
133 import org.graalvm.compiler.nodes.memory.VolatileReadNode; |
132 import org.graalvm.compiler.nodes.memory.WriteNode; |
134 import org.graalvm.compiler.nodes.memory.WriteNode; |
133 import org.graalvm.compiler.nodes.memory.address.AddressNode; |
135 import org.graalvm.compiler.nodes.memory.address.AddressNode; |
134 import org.graalvm.compiler.nodes.memory.address.IndexAddressNode; |
136 import org.graalvm.compiler.nodes.memory.address.IndexAddressNode; |
135 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode; |
137 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode; |
136 import org.graalvm.compiler.nodes.spi.Lowerable; |
138 import org.graalvm.compiler.nodes.spi.Lowerable; |
395 Stamp loadStamp = loadStamp(loadField.stamp(NodeView.DEFAULT), getStorageKind(field)); |
397 Stamp loadStamp = loadStamp(loadField.stamp(NodeView.DEFAULT), getStorageKind(field)); |
396 |
398 |
397 AddressNode address = createFieldAddress(graph, object, field); |
399 AddressNode address = createFieldAddress(graph, object, field); |
398 assert address != null : "Field that is loaded must not be eliminated: " + field.getDeclaringClass().toJavaName(true) + "." + field.getName(); |
400 assert address != null : "Field that is loaded must not be eliminated: " + field.getDeclaringClass().toJavaName(true) + "." + field.getName(); |
399 |
401 |
400 ReadNode memoryRead = graph.add(new ReadNode(address, fieldLocationIdentity(field), loadStamp, fieldLoadBarrierType(field))); |
402 ReadNode memoryRead = null; |
|
403 if (loadField.isVolatile() && GraalOptions.LateMembars.getValue(graph.getOptions())) { |
|
404 memoryRead = graph.add(new VolatileReadNode(address, fieldLocationIdentity(field), loadStamp, fieldLoadBarrierType(field))); |
|
405 } else { |
|
406 memoryRead = graph.add(new ReadNode(address, fieldLocationIdentity(field), loadStamp, fieldLoadBarrierType(field))); |
|
407 } |
401 ValueNode readValue = implicitLoadConvert(graph, getStorageKind(field), memoryRead); |
408 ValueNode readValue = implicitLoadConvert(graph, getStorageKind(field), memoryRead); |
402 loadField.replaceAtUsages(readValue); |
409 loadField.replaceAtUsages(readValue); |
403 graph.replaceFixed(loadField, memoryRead); |
410 graph.replaceFixed(loadField, memoryRead); |
404 |
411 |
405 if (loadField.isVolatile()) { |
412 if (loadField.isVolatile() && !GraalOptions.LateMembars.getValue(graph.getOptions())) { |
406 MembarNode preMembar = graph.add(new MembarNode(JMM_PRE_VOLATILE_READ)); |
413 MembarNode preMembar = graph.add(new MembarNode(JMM_PRE_VOLATILE_READ)); |
407 graph.addBeforeFixed(memoryRead, preMembar); |
414 graph.addBeforeFixed(memoryRead, preMembar); |
408 MembarNode postMembar = graph.add(new MembarNode(JMM_POST_VOLATILE_READ)); |
415 MembarNode postMembar = graph.add(new MembarNode(JMM_POST_VOLATILE_READ)); |
409 graph.addAfterFixed(memoryRead, postMembar); |
416 graph.addAfterFixed(memoryRead, postMembar); |
410 } |
417 } |
417 object = createNullCheckedValue(object, storeField, tool); |
424 object = createNullCheckedValue(object, storeField, tool); |
418 ValueNode value = implicitStoreConvert(graph, getStorageKind(storeField.field()), storeField.value()); |
425 ValueNode value = implicitStoreConvert(graph, getStorageKind(storeField.field()), storeField.value()); |
419 AddressNode address = createFieldAddress(graph, object, field); |
426 AddressNode address = createFieldAddress(graph, object, field); |
420 assert address != null; |
427 assert address != null; |
421 |
428 |
422 WriteNode memoryWrite = graph.add(new WriteNode(address, fieldLocationIdentity(field), value, fieldStoreBarrierType(storeField.field()))); |
429 WriteNode memoryWrite = graph.add(new WriteNode(address, fieldLocationIdentity(field), value, fieldStoreBarrierType(storeField.field()), storeField.isVolatile())); |
423 memoryWrite.setStateAfter(storeField.stateAfter()); |
430 memoryWrite.setStateAfter(storeField.stateAfter()); |
424 graph.replaceFixedWithFixed(storeField, memoryWrite); |
431 graph.replaceFixedWithFixed(storeField, memoryWrite); |
425 |
432 |
426 if (storeField.isVolatile()) { |
433 if (storeField.isVolatile() && !GraalOptions.LateMembars.getValue(graph.getOptions())) { |
427 MembarNode preMembar = graph.add(new MembarNode(JMM_PRE_VOLATILE_WRITE)); |
434 MembarNode preMembar = graph.add(new MembarNode(JMM_PRE_VOLATILE_WRITE)); |
428 graph.addBeforeFixed(memoryWrite, preMembar); |
435 graph.addBeforeFixed(memoryWrite, preMembar); |
429 MembarNode postMembar = graph.add(new MembarNode(JMM_POST_VOLATILE_WRITE)); |
436 MembarNode postMembar = graph.add(new MembarNode(JMM_POST_VOLATILE_WRITE)); |
430 graph.addAfterFixed(memoryWrite, postMembar); |
437 graph.addAfterFixed(memoryWrite, postMembar); |
431 } |
438 } |
526 } |
533 } |
527 } |
534 } |
528 |
535 |
529 AddressNode address = createArrayIndexAddress(graph, array, elementKind, storeIndexed.index(), boundsCheck); |
536 AddressNode address = createArrayIndexAddress(graph, array, elementKind, storeIndexed.index(), boundsCheck); |
530 WriteNode memoryWrite = graph.add(new WriteNode(address, NamedLocationIdentity.getArrayLocation(elementKind), implicitStoreConvert(graph, elementKind, value), |
537 WriteNode memoryWrite = graph.add(new WriteNode(address, NamedLocationIdentity.getArrayLocation(elementKind), implicitStoreConvert(graph, elementKind, value), |
531 arrayStoreBarrierType(storeIndexed.elementKind()))); |
538 arrayStoreBarrierType(storeIndexed.elementKind()), false)); |
532 memoryWrite.setGuard(boundsCheck); |
539 memoryWrite.setGuard(boundsCheck); |
533 if (condition != null) { |
540 if (condition != null) { |
534 tool.createGuard(storeIndexed, condition, DeoptimizationReason.ArrayStoreException, DeoptimizationAction.InvalidateReprofile); |
541 tool.createGuard(storeIndexed, condition, DeoptimizationReason.ArrayStoreException, DeoptimizationAction.InvalidateReprofile); |
535 } |
542 } |
536 memoryWrite.setStateAfter(storeIndexed.stateAfter()); |
543 memoryWrite.setStateAfter(storeIndexed.stateAfter()); |
621 ValueNode expectedValue = implicitStoreConvert(graph, valueKind, cas.expected()); |
628 ValueNode expectedValue = implicitStoreConvert(graph, valueKind, cas.expected()); |
622 ValueNode newValue = implicitStoreConvert(graph, valueKind, cas.newValue()); |
629 ValueNode newValue = implicitStoreConvert(graph, valueKind, cas.newValue()); |
623 |
630 |
624 AddressNode address = graph.unique(new OffsetAddressNode(cas.object(), cas.offset())); |
631 AddressNode address = graph.unique(new OffsetAddressNode(cas.object(), cas.offset())); |
625 BarrierType barrierType = guessStoreBarrierType(cas.object(), expectedValue); |
632 BarrierType barrierType = guessStoreBarrierType(cas.object(), expectedValue); |
626 LogicCompareAndSwapNode atomicNode = graph.add(new LogicCompareAndSwapNode(address, cas.getLocationIdentity(), expectedValue, newValue, barrierType)); |
633 LogicCompareAndSwapNode atomicNode = graph.add(new LogicCompareAndSwapNode(address, cas.getKilledLocationIdentity(), expectedValue, newValue, barrierType)); |
627 atomicNode.setStateAfter(cas.stateAfter()); |
634 atomicNode.setStateAfter(cas.stateAfter()); |
628 graph.replaceFixedWithFixed(cas, atomicNode); |
635 graph.replaceFixedWithFixed(cas, atomicNode); |
629 } |
636 } |
630 |
637 |
631 protected void lowerCompareAndExchangeNode(UnsafeCompareAndExchangeNode cas) { |
638 protected void lowerCompareAndExchangeNode(UnsafeCompareAndExchangeNode cas) { |
635 ValueNode expectedValue = implicitStoreConvert(graph, valueKind, cas.expected()); |
642 ValueNode expectedValue = implicitStoreConvert(graph, valueKind, cas.expected()); |
636 ValueNode newValue = implicitStoreConvert(graph, valueKind, cas.newValue()); |
643 ValueNode newValue = implicitStoreConvert(graph, valueKind, cas.newValue()); |
637 |
644 |
638 AddressNode address = graph.unique(new OffsetAddressNode(cas.object(), cas.offset())); |
645 AddressNode address = graph.unique(new OffsetAddressNode(cas.object(), cas.offset())); |
639 BarrierType barrierType = guessStoreBarrierType(cas.object(), expectedValue); |
646 BarrierType barrierType = guessStoreBarrierType(cas.object(), expectedValue); |
640 ValueCompareAndSwapNode atomicNode = graph.add(new ValueCompareAndSwapNode(address, expectedValue, newValue, cas.getLocationIdentity(), barrierType)); |
647 ValueCompareAndSwapNode atomicNode = graph.add(new ValueCompareAndSwapNode(address, expectedValue, newValue, cas.getKilledLocationIdentity(), barrierType)); |
641 ValueNode coercedNode = implicitLoadConvert(graph, valueKind, atomicNode, true); |
648 ValueNode coercedNode = implicitLoadConvert(graph, valueKind, atomicNode, true); |
642 atomicNode.setStateAfter(cas.stateAfter()); |
649 atomicNode.setStateAfter(cas.stateAfter()); |
643 cas.replaceAtUsages(coercedNode); |
650 cas.replaceAtUsages(coercedNode); |
644 graph.replaceFixedWithFixed(cas, atomicNode); |
651 graph.replaceFixedWithFixed(cas, atomicNode); |
645 } |
652 } |
651 ValueNode newValue = implicitStoreConvert(graph, valueKind, n.newValue()); |
658 ValueNode newValue = implicitStoreConvert(graph, valueKind, n.newValue()); |
652 |
659 |
653 AddressNode address = graph.unique(new OffsetAddressNode(n.object(), n.offset())); |
660 AddressNode address = graph.unique(new OffsetAddressNode(n.object(), n.offset())); |
654 BarrierType barrierType = guessStoreBarrierType(n.object(), n.newValue()); |
661 BarrierType barrierType = guessStoreBarrierType(n.object(), n.newValue()); |
655 LIRKind lirAccessKind = LIRKind.fromJavaKind(target.arch, valueKind); |
662 LIRKind lirAccessKind = LIRKind.fromJavaKind(target.arch, valueKind); |
656 LoweredAtomicReadAndWriteNode memoryRead = graph.add(new LoweredAtomicReadAndWriteNode(address, n.getLocationIdentity(), newValue, lirAccessKind, barrierType)); |
663 LoweredAtomicReadAndWriteNode memoryRead = graph.add(new LoweredAtomicReadAndWriteNode(address, n.getKilledLocationIdentity(), newValue, lirAccessKind, barrierType)); |
657 memoryRead.setStateAfter(n.stateAfter()); |
664 memoryRead.setStateAfter(n.stateAfter()); |
658 |
665 |
659 ValueNode readValue = implicitLoadConvert(graph, valueKind, memoryRead); |
666 ValueNode readValue = implicitLoadConvert(graph, valueKind, memoryRead); |
660 n.stateAfter().replaceFirstInput(n, memoryRead); |
667 n.stateAfter().replaceFirstInput(n, memoryRead); |
661 n.replaceAtUsages(readValue); |
668 n.replaceAtUsages(readValue); |
742 StructuredGraph graph = store.graph(); |
749 StructuredGraph graph = store.graph(); |
743 boolean compressible = store.value().getStackKind() == JavaKind.Object; |
750 boolean compressible = store.value().getStackKind() == JavaKind.Object; |
744 JavaKind valueKind = store.accessKind(); |
751 JavaKind valueKind = store.accessKind(); |
745 ValueNode value = implicitStoreConvert(graph, valueKind, store.value(), compressible); |
752 ValueNode value = implicitStoreConvert(graph, valueKind, store.value(), compressible); |
746 AddressNode address = createUnsafeAddress(graph, store.object(), store.offset()); |
753 AddressNode address = createUnsafeAddress(graph, store.object(), store.offset()); |
747 WriteNode write = graph.add(new WriteNode(address, store.getLocationIdentity(), value, unsafeStoreBarrierType(store))); |
754 WriteNode write = graph.add(new WriteNode(address, store.getKilledLocationIdentity(), value, unsafeStoreBarrierType(store), false)); |
748 write.setStateAfter(store.stateAfter()); |
755 write.setStateAfter(store.stateAfter()); |
749 graph.replaceFixedWithFixed(store, write); |
756 graph.replaceFixedWithFixed(store, write); |
750 } |
757 } |
751 |
758 |
752 protected void lowerUnsafeMemoryStoreNode(UnsafeMemoryStoreNode store) { |
759 protected void lowerUnsafeMemoryStoreNode(UnsafeMemoryStoreNode store) { |
753 StructuredGraph graph = store.graph(); |
760 StructuredGraph graph = store.graph(); |
754 assert store.getValue().getStackKind() != JavaKind.Object; |
761 assert store.getValue().getStackKind() != JavaKind.Object; |
755 JavaKind valueKind = store.getKind(); |
762 JavaKind valueKind = store.getKind(); |
756 ValueNode value = implicitStoreConvert(graph, valueKind, store.getValue(), false); |
763 ValueNode value = implicitStoreConvert(graph, valueKind, store.getValue(), false); |
757 AddressNode address = graph.addOrUniqueWithInputs(OffsetAddressNode.create(store.getAddress())); |
764 AddressNode address = graph.addOrUniqueWithInputs(OffsetAddressNode.create(store.getAddress())); |
758 WriteNode write = graph.add(new WriteNode(address, store.getLocationIdentity(), value, BarrierType.NONE)); |
765 WriteNode write = graph.add(new WriteNode(address, store.getKilledLocationIdentity(), value, BarrierType.NONE, false)); |
759 write.setStateAfter(store.stateAfter()); |
766 write.setStateAfter(store.stateAfter()); |
760 graph.replaceFixedWithFixed(store, write); |
767 graph.replaceFixedWithFixed(store, write); |
761 } |
768 } |
762 |
769 |
763 protected void lowerJavaReadNode(JavaReadNode read) { |
770 protected void lowerJavaReadNode(JavaReadNode read) { |
780 } |
787 } |
781 |
788 |
782 protected void lowerJavaWriteNode(JavaWriteNode write) { |
789 protected void lowerJavaWriteNode(JavaWriteNode write) { |
783 StructuredGraph graph = write.graph(); |
790 StructuredGraph graph = write.graph(); |
784 ValueNode value = implicitStoreConvert(graph, write.getWriteKind(), write.value(), write.isCompressible()); |
791 ValueNode value = implicitStoreConvert(graph, write.getWriteKind(), write.value(), write.isCompressible()); |
785 WriteNode memoryWrite = graph.add(new WriteNode(write.getAddress(), write.getLocationIdentity(), value, write.getBarrierType())); |
792 WriteNode memoryWrite = graph.add(new WriteNode(write.getAddress(), write.getKilledLocationIdentity(), value, write.getBarrierType(), false)); |
786 memoryWrite.setStateAfter(write.stateAfter()); |
793 memoryWrite.setStateAfter(write.stateAfter()); |
787 graph.replaceFixedWithFixed(write, memoryWrite); |
794 graph.replaceFixedWithFixed(write, memoryWrite); |
788 memoryWrite.setGuard(write.getGuard()); |
795 memoryWrite.setGuard(write.getGuard()); |
789 } |
796 } |
790 |
797 |
840 } else { |
847 } else { |
841 address = createOffsetAddress(graph, newObject, metaAccess.getArrayBaseOffset(entryKind) + i * metaAccess.getArrayIndexScale(entryKind)); |
848 address = createOffsetAddress(graph, newObject, metaAccess.getArrayBaseOffset(entryKind) + i * metaAccess.getArrayIndexScale(entryKind)); |
842 barrierType = arrayInitializationBarrier(entryKind); |
849 barrierType = arrayInitializationBarrier(entryKind); |
843 } |
850 } |
844 if (address != null) { |
851 if (address != null) { |
845 WriteNode write = new WriteNode(address, LocationIdentity.init(), implicitStoreConvert(graph, entryKind, value), barrierType); |
852 WriteNode write = new WriteNode(address, LocationIdentity.init(), implicitStoreConvert(graph, entryKind, value), barrierType, false); |
846 graph.addAfterFixed(newObject, graph.add(write)); |
853 graph.addAfterFixed(newObject, graph.add(write)); |
847 } |
854 } |
848 } |
855 } |
849 valuePos++; |
856 valuePos++; |
850 } |
857 } |
873 } else { |
880 } else { |
874 address = createArrayAddress(graph, newObject, virtual.entryKind(i), ConstantNode.forInt(i, graph)); |
881 address = createArrayAddress(graph, newObject, virtual.entryKind(i), ConstantNode.forInt(i, graph)); |
875 barrierType = arrayStoreBarrierType(virtual.entryKind(i)); |
882 barrierType = arrayStoreBarrierType(virtual.entryKind(i)); |
876 } |
883 } |
877 if (address != null) { |
884 if (address != null) { |
878 WriteNode write = new WriteNode(address, LocationIdentity.init(), implicitStoreConvert(graph, JavaKind.Object, allocValue), barrierType); |
885 WriteNode write = new WriteNode(address, LocationIdentity.init(), implicitStoreConvert(graph, JavaKind.Object, allocValue), barrierType, false); |
879 graph.addBeforeFixed(commit, graph.add(write)); |
886 graph.addBeforeFixed(commit, graph.add(write)); |
880 } |
887 } |
881 } |
888 } |
882 } |
889 } |
883 valuePos++; |
890 valuePos++; |