--- a/src/hotspot/share/opto/memnode.cpp Wed Aug 22 15:10:40 2018 +0800
+++ b/src/hotspot/share/opto/memnode.cpp Tue Aug 14 16:54:47 2018 +0200
@@ -2521,45 +2521,63 @@
Node* adr = in(MemNode::Address);
Node* val = in(MemNode::ValueIn);
+ Node* result = this;
+
// Load then Store? Then the Store is useless
if (val->is_Load() &&
val->in(MemNode::Address)->eqv_uncast(adr) &&
val->in(MemNode::Memory )->eqv_uncast(mem) &&
val->as_Load()->store_Opcode() == Opcode()) {
- return mem;
+ result = mem;
}
// Two stores in a row of the same value?
- if (mem->is_Store() &&
+ if (result == this &&
+ mem->is_Store() &&
mem->in(MemNode::Address)->eqv_uncast(adr) &&
mem->in(MemNode::ValueIn)->eqv_uncast(val) &&
mem->Opcode() == Opcode()) {
- return mem;
+ result = mem;
}
// Store of zero anywhere into a freshly-allocated object?
// Then the store is useless.
// (It must already have been captured by the InitializeNode.)
- if (ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
+ if (result == this &&
+ ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
// a newly allocated object is already all-zeroes everywhere
if (mem->is_Proj() && mem->in(0)->is_Allocate()) {
- return mem;
+ result = mem;
}
- // the store may also apply to zero-bits in an earlier object
- Node* prev_mem = find_previous_store(phase);
- // Steps (a), (b): Walk past independent stores to find an exact match.
- if (prev_mem != NULL) {
- Node* prev_val = can_see_stored_value(prev_mem, phase);
- if (prev_val != NULL && phase->eqv(prev_val, val)) {
- // prev_val and val might differ by a cast; it would be good
- // to keep the more informative of the two.
- return mem;
+ if (result == this) {
+ // the store may also apply to zero-bits in an earlier object
+ Node* prev_mem = find_previous_store(phase);
+ // Steps (a), (b): Walk past independent stores to find an exact match.
+ if (prev_mem != NULL) {
+ Node* prev_val = can_see_stored_value(prev_mem, phase);
+ if (prev_val != NULL && phase->eqv(prev_val, val)) {
+ // prev_val and val might differ by a cast; it would be good
+ // to keep the more informative of the two.
+ result = mem;
+ }
}
}
}
- return this;
+ if (result != this && phase->is_IterGVN() != NULL) {
+ MemBarNode* trailing = trailing_membar();
+ if (trailing != NULL) {
+#ifdef ASSERT
+ const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
+ assert(t_oop == NULL || t_oop->is_known_instance_field(), "only for non escaping objects");
+#endif
+ PhaseIterGVN* igvn = phase->is_IterGVN();
+ trailing->remove(igvn);
+ }
+ }
+
+ return result;
}
//------------------------------match_edge-------------------------------------
@@ -2637,6 +2655,33 @@
return true;
}
+MemBarNode* StoreNode::trailing_membar() const {
+ if (is_release()) {
+ MemBarNode* trailing_mb = NULL;
+ for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
+ Node* u = fast_out(i);
+ if (u->is_MemBar()) {
+ if (u->as_MemBar()->trailing_store()) {
+ assert(u->Opcode() == Op_MemBarVolatile, "");
+ assert(trailing_mb == NULL, "only one");
+ trailing_mb = u->as_MemBar();
+#ifdef ASSERT
+ Node* leading = u->as_MemBar()->leading_membar();
+ assert(leading->Opcode() == Op_MemBarRelease, "incorrect membar");
+ assert(leading->as_MemBar()->leading_store(), "incorrect membar pair");
+ assert(leading->as_MemBar()->trailing_membar() == u, "incorrect membar pair");
+#endif
+ } else {
+ assert(u->as_MemBar()->standalone(), "");
+ }
+ }
+ }
+ return trailing_mb;
+ }
+ return NULL;
+}
+
+
//=============================================================================
//------------------------------Ideal------------------------------------------
// If the store is from an AND mask that leaves the low bits untouched, then
@@ -2749,6 +2794,30 @@
return true;
}
+MemBarNode* LoadStoreNode::trailing_membar() const {
+ MemBarNode* trailing = NULL;
+ for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
+ Node* u = fast_out(i);
+ if (u->is_MemBar()) {
+ if (u->as_MemBar()->trailing_load_store()) {
+ assert(u->Opcode() == Op_MemBarAcquire, "");
+ assert(trailing == NULL, "only one");
+ trailing = u->as_MemBar();
+#ifdef ASSERT
+ Node* leading = trailing->leading_membar();
+ assert(support_IRIW_for_not_multiple_copy_atomic_cpu || leading->Opcode() == Op_MemBarRelease, "incorrect membar");
+ assert(leading->as_MemBar()->leading_load_store(), "incorrect membar pair");
+ assert(leading->as_MemBar()->trailing_membar() == trailing, "incorrect membar pair");
+#endif
+ } else {
+ assert(u->as_MemBar()->standalone(), "wrong barrier kind");
+ }
+ }
+ }
+
+ return trailing;
+}
+
uint LoadStoreNode::size_of() const { return sizeof(*this); }
//=============================================================================
@@ -2934,7 +3003,10 @@
//=============================================================================
MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
: MultiNode(TypeFunc::Parms + (precedent == NULL? 0: 1)),
- _adr_type(C->get_adr_type(alias_idx))
+ _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
+#ifdef ASSERT
+ , _pair_idx(0)
+#endif
{
init_class_id(Class_MemBar);
Node* top = C->top();
@@ -2969,6 +3041,21 @@
}
}
+void MemBarNode::remove(PhaseIterGVN *igvn) {
+ if (outcnt() != 2) {
+ return;
+ }
+ if (trailing_store() || trailing_load_store()) {
+ MemBarNode* leading = leading_membar();
+ if (leading != NULL) {
+ assert(leading->trailing_membar() == this, "inconsistent leading/trailing membars");
+ leading->remove(igvn);
+ }
+ }
+ igvn->replace_node(proj_out(TypeFunc::Memory), in(TypeFunc::Memory));
+ igvn->replace_node(proj_out(TypeFunc::Control), in(TypeFunc::Control));
+}
+
//------------------------------Ideal------------------------------------------
// Return a node which is more "ideal" than the current node. Strip out
// control copies
@@ -3035,8 +3122,7 @@
if (eliminate) {
// Replace MemBar projections by its inputs.
PhaseIterGVN* igvn = phase->is_IterGVN();
- igvn->replace_node(proj_out(TypeFunc::Memory), in(TypeFunc::Memory));
- igvn->replace_node(proj_out(TypeFunc::Control), in(TypeFunc::Control));
+ remove(igvn);
// Must return either the original node (now dead) or a new node
// (Do not return a top here, since that would break the uniqueness of top.)
return new ConINode(TypeInt::ZERO);
@@ -3065,6 +3151,98 @@
return NULL;
}
+void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
+ trailing->_kind = TrailingStore;
+ leading->_kind = LeadingStore;
+#ifdef ASSERT
+ trailing->_pair_idx = leading->_idx;
+ leading->_pair_idx = leading->_idx;
+#endif
+}
+
+void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
+ trailing->_kind = TrailingLoadStore;
+ leading->_kind = LeadingLoadStore;
+#ifdef ASSERT
+ trailing->_pair_idx = leading->_idx;
+ leading->_pair_idx = leading->_idx;
+#endif
+}
+
+MemBarNode* MemBarNode::trailing_membar() const {
+ Node* trailing = (Node*)this;
+ VectorSet seen(Thread::current()->resource_area());
+ while (!trailing->is_MemBar() || !trailing->as_MemBar()->trailing()) {
+ if (seen.test_set(trailing->_idx)) {
+ // Dying subgraph?
+ return NULL;
+ }
+ for (DUIterator_Fast jmax, j = trailing->fast_outs(jmax); j < jmax; j++) {
+ Node* next = trailing->fast_out(j);
+ if (next != trailing && next->is_CFG()) {
+ trailing = next;
+ break;
+ }
+ }
+ }
+ MemBarNode* mb = trailing->as_MemBar();
+ assert((mb->_kind == TrailingStore && _kind == LeadingStore) ||
+ (mb->_kind == TrailingLoadStore && _kind == LeadingLoadStore), "bad trailing membar");
+ assert(mb->_pair_idx == _pair_idx, "bad trailing membar");
+ return mb;
+}
+
+MemBarNode* MemBarNode::leading_membar() const {
+ VectorSet seen(Thread::current()->resource_area());
+ Node* leading = in(0);
+ while (leading != NULL && (!leading->is_MemBar() || !leading->as_MemBar()->leading())) {
+ if (seen.test_set(leading->_idx)) {
+ // Dying subgraph?
+ return NULL;
+ }
+ if (leading->is_Region()) {
+ leading = leading->in(1);
+ } else {
+ leading = leading->in(0);
+ }
+ }
+#ifdef ASSERT
+ Unique_Node_List wq;
+ wq.push((Node*)this);
+ uint found = 0;
+ for (uint i = 0; i < wq.size(); i++) {
+ Node* n = wq.at(i);
+ if (n->is_Region()) {
+ for (uint j = 1; j < n->req(); j++) {
+ Node* in = n->in(j);
+ if (in != NULL && !in->is_top()) {
+ wq.push(in);
+ }
+ }
+ } else {
+ if (n->is_MemBar() && n->as_MemBar()->leading()) {
+ assert(n == leading, "consistency check failed");
+ found++;
+ } else {
+ Node* in = n->in(0);
+ if (in != NULL && !in->is_top()) {
+ wq.push(in);
+ }
+ }
+ }
+ }
+ assert(found == 1 || (found == 0 && leading == NULL), "consistency check failed");
+#endif
+ if (leading == NULL) {
+ return NULL;
+ }
+ MemBarNode* mb = leading->as_MemBar();
+ assert((mb->_kind == LeadingStore && _kind == TrailingStore) ||
+ (mb->_kind == LeadingLoadStore && _kind == TrailingLoadStore), "bad leading membar");
+ assert(mb->_pair_idx == _pair_idx, "bad leading membar");
+ return mb;
+}
+
//===========================InitializeNode====================================
// SUMMARY:
// This node acts as a memory barrier on raw memory, after some raw stores.