--- a/hotspot/src/share/vm/opto/compile.cpp Fri Feb 18 10:07:34 2011 -0800
+++ b/hotspot/src/share/vm/opto/compile.cpp Tue Apr 05 19:14:03 2011 -0700
@@ -2050,6 +2050,52 @@
// Note that OffsetBot and OffsetTop are very negative.
}
+// Eliminate trivially redundant StoreCMs and accumulate their
+// precedence edges.
+static void eliminate_redundant_card_marks(Node* n) {
+ assert(n->Opcode() == Op_StoreCM, "expected StoreCM");
+ if (n->in(MemNode::Address)->outcnt() > 1) {
+ // There are multiple users of the same address so it might be
+ // possible to eliminate some of the StoreCMs
+ Node* mem = n->in(MemNode::Memory);
+ Node* adr = n->in(MemNode::Address);
+ Node* val = n->in(MemNode::ValueIn);
+ Node* prev = n;
+ bool done = false;
+ // Walk the chain of StoreCMs eliminating ones that match. As
+ // long as it's a chain of single users then the optimization is
+ // safe. Eliminating partially redundant StoreCMs would require
+ // cloning copies down the other paths.
+ while (mem->Opcode() == Op_StoreCM && mem->outcnt() == 1 && !done) {
+ if (adr == mem->in(MemNode::Address) &&
+ val == mem->in(MemNode::ValueIn)) {
+ // redundant StoreCM
+ if (mem->req() > MemNode::OopStore) {
+ // Hasn't been processed by this code yet.
+ n->add_prec(mem->in(MemNode::OopStore));
+ } else {
+ // Already converted to precedence edge
+ for (uint i = mem->req(); i < mem->len(); i++) {
+ // Accumulate any precedence edges
+ if (mem->in(i) != NULL) {
+ n->add_prec(mem->in(i));
+ }
+ }
+ // Everything above this point has been processed.
+ done = true;
+ }
+ // Eliminate the previous StoreCM
+ prev->set_req(MemNode::Memory, mem->in(MemNode::Memory));
+ assert(mem->outcnt() == 0, "should be dead");
+ mem->disconnect_inputs(NULL);
+ } else {
+ prev = mem;
+ }
+ mem = prev->in(MemNode::Memory);
+ }
+ }
+}
+
//------------------------------final_graph_reshaping_impl----------------------
// Implement items 1-5 from final_graph_reshaping below.
static void final_graph_reshaping_impl( Node *n, Final_Reshape_Counts &frc ) {
@@ -2176,9 +2222,19 @@
frc.inc_float_count();
goto handle_mem;
+ case Op_StoreCM:
+ {
+ // Convert OopStore dependence into precedence edge
+ Node* prec = n->in(MemNode::OopStore);
+ n->del_req(MemNode::OopStore);
+ n->add_prec(prec);
+ eliminate_redundant_card_marks(n);
+ }
+
+ // fall through
+
case Op_StoreB:
case Op_StoreC:
- case Op_StoreCM:
case Op_StorePConditional:
case Op_StoreI:
case Op_StoreL:
--- a/hotspot/src/share/vm/opto/lcm.cpp Fri Feb 18 10:07:34 2011 -0800
+++ b/hotspot/src/share/vm/opto/lcm.cpp Tue Apr 05 19:14:03 2011 -0700
@@ -685,20 +685,22 @@
}
ready_cnt[n->_idx] = local; // Count em up
- // A few node types require changing a required edge to a precedence edge
- // before allocation.
+#ifdef ASSERT
if( UseConcMarkSweepGC || UseG1GC ) {
if( n->is_Mach() && n->as_Mach()->ideal_Opcode() == Op_StoreCM ) {
- // Note: Required edges with an index greater than oper_input_base
- // are not supported by the allocator.
- // Note2: Can only depend on unmatched edge being last,
- // can not depend on its absolute position.
- Node *oop_store = n->in(n->req() - 1);
- n->del_req(n->req() - 1);
- n->add_prec(oop_store);
- assert(cfg->_bbs[oop_store->_idx]->_dom_depth <= this->_dom_depth, "oop_store must dominate card-mark");
+ // Check the precedence edges
+ for (uint prec = n->req(); prec < n->len(); prec++) {
+ Node* oop_store = n->in(prec);
+ if (oop_store != NULL) {
+ assert(cfg->_bbs[oop_store->_idx]->_dom_depth <= this->_dom_depth, "oop_store must dominate card-mark");
+ }
+ }
}
}
+#endif
+
+ // A few node types require changing a required edge to a precedence edge
+ // before allocation.
if( n->is_Mach() && n->req() > TypeFunc::Parms &&
(n->as_Mach()->ideal_Opcode() == Op_MemBarAcquire ||
n->as_Mach()->ideal_Opcode() == Op_MemBarVolatile) ) {
--- a/hotspot/src/share/vm/opto/memnode.cpp Fri Feb 18 10:07:34 2011 -0800
+++ b/hotspot/src/share/vm/opto/memnode.cpp Tue Apr 05 19:14:03 2011 -0700
@@ -2159,9 +2159,12 @@
Node* mem = in(MemNode::Memory);
Node* address = in(MemNode::Address);
- // Back-to-back stores to same address? Fold em up.
- // Generally unsafe if I have intervening uses...
- if (mem->is_Store() && phase->eqv_uncast(mem->in(MemNode::Address), address)) {
+ // Back-to-back stores to same address? Fold em up. Generally
+ // unsafe if I have intervening uses... Also disallowed for StoreCM
+ // since they must follow each StoreP operation. Redundant StoreCMs
+ // are eliminated just before matching in final_graph_reshape.
+ if (mem->is_Store() && phase->eqv_uncast(mem->in(MemNode::Address), address) &&
+ mem->Opcode() != Op_StoreCM) {
// Looking at a dead closed cycle of memory?
assert(mem != mem->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
--- a/hotspot/src/share/vm/opto/output.cpp Fri Feb 18 10:07:34 2011 -0800
+++ b/hotspot/src/share/vm/opto/output.cpp Tue Apr 05 19:14:03 2011 -0700
@@ -1354,15 +1354,20 @@
// Check that oop-store precedes the card-mark
else if( mach->ideal_Opcode() == Op_StoreCM ) {
uint storeCM_idx = j;
- Node *oop_store = mach->in(mach->_cnt); // First precedence edge
- assert( oop_store != NULL, "storeCM expects a precedence edge");
- uint i4;
- for( i4 = 0; i4 < last_inst; ++i4 ) {
- if( b->_nodes[i4] == oop_store ) break;
+ int count = 0;
+ for (uint prec = mach->req(); prec < mach->len(); prec++) {
+ Node *oop_store = mach->in(prec); // Precedence edge
+ if (oop_store == NULL) continue;
+ count++;
+ uint i4;
+ for( i4 = 0; i4 < last_inst; ++i4 ) {
+ if( b->_nodes[i4] == oop_store ) break;
+ }
+ // Note: This test can provide a false failure if other precedence
+ // edges have been added to the storeCMNode.
+ assert( i4 == last_inst || i4 < storeCM_idx, "CM card-mark executes before oop-store");
}
- // Note: This test can provide a false failure if other precedence
- // edges have been added to the storeCMNode.
- assert( i4 == last_inst || i4 < storeCM_idx, "CM card-mark executes before oop-store");
+ assert(count > 0, "storeCM expects at least one precedence edge");
}
#endif