--- a/hotspot/src/share/vm/opto/memnode.cpp Mon May 11 09:44:07 2015 +0200
+++ b/hotspot/src/share/vm/opto/memnode.cpp Tue May 12 10:27:50 2015 +0200
@@ -28,6 +28,7 @@
#include "memory/allocation.inline.hpp"
#include "oops/objArrayKlass.hpp"
#include "opto/addnode.hpp"
+#include "opto/arraycopynode.hpp"
#include "opto/cfgnode.hpp"
#include "opto/compile.hpp"
#include "opto/connode.hpp"
@@ -107,6 +108,32 @@
#endif
+static bool membar_for_arraycopy_helper(const TypeOopPtr *t_oop, MergeMemNode* mm, PhaseTransform *phase) {
+ if (mm->memory_at(Compile::AliasIdxRaw)->is_Proj()) {
+ Node* n = mm->memory_at(Compile::AliasIdxRaw)->in(0);
+ if ((n->is_ArrayCopy() && n->as_ArrayCopy()->may_modify(t_oop, phase)) ||
+ (n->is_CallLeaf() && n->as_CallLeaf()->may_modify(t_oop, phase))) {
+ return true;
+ }
+ }
+ return false;
+}
+
+static bool membar_for_arraycopy(const TypeOopPtr *t_oop, MemBarNode* mb, PhaseTransform *phase) {
+ Node* mem = mb->in(TypeFunc::Memory);
+ if (mem->is_MergeMem()) {
+ return membar_for_arraycopy_helper(t_oop, mem->as_MergeMem(), phase);
+ } else if (mem->is_Phi()) {
+ // after macro expansion of an ArrayCopyNode we may have a Phi
+ for (uint i = 1; i < mem->req(); i++) {
+ if (mem->in(i) != NULL && mem->in(i)->is_MergeMem() && membar_for_arraycopy_helper(t_oop, mem->in(i)->as_MergeMem(), phase)) {
+ return true;
+ }
+ }
+ }
+ return false;
+}
+
Node *MemNode::optimize_simple_memory_chain(Node *mchain, const TypeOopPtr *t_oop, Node *load, PhaseGVN *phase) {
assert((t_oop != NULL), "sanity");
bool is_instance = t_oop->is_known_instance_field();
@@ -129,6 +156,7 @@
if (proj_in->is_Allocate() && proj_in->_idx == instance_id) {
break; // hit one of our sentinels
} else if (proj_in->is_Call()) {
+ // ArrayCopyNodes processed here as well
CallNode *call = proj_in->as_Call();
if (!call->may_modify(t_oop, phase)) { // returns false for instances
result = call->in(TypeFunc::Memory);
@@ -136,7 +164,7 @@
} else if (proj_in->is_Initialize()) {
AllocateNode* alloc = proj_in->as_Initialize()->allocation();
// Stop if this is the initialization for the object instance which
- // which contains this memory slice, otherwise skip over it.
+ // contains this memory slice, otherwise skip over it.
if ((alloc == NULL) || (alloc->_idx == instance_id)) {
break;
}
@@ -150,6 +178,9 @@
}
}
} else if (proj_in->is_MemBar()) {
+ if (membar_for_arraycopy(t_oop, proj_in->as_MemBar(), phase)) {
+ break;
+ }
result = proj_in->in(TypeFunc::Memory);
} else {
assert(false, "unexpected projection");
@@ -477,6 +508,75 @@
}
+// Find an arraycopy that must have set (can_see_stored_value=true) or
+// could have set (can_see_stored_value=false) the value for this load
+Node* LoadNode::find_previous_arraycopy(PhaseTransform* phase, Node* ld_alloc, Node*& mem, bool can_see_stored_value) const {
+ if (mem->is_Proj() && mem->in(0) != NULL && (mem->in(0)->Opcode() == Op_MemBarStoreStore ||
+ mem->in(0)->Opcode() == Op_MemBarCPUOrder)) {
+ Node* mb = mem->in(0);
+ if (mb->in(0) != NULL && mb->in(0)->is_Proj() &&
+ mb->in(0)->in(0) != NULL && mb->in(0)->in(0)->is_ArrayCopy()) {
+ ArrayCopyNode* ac = mb->in(0)->in(0)->as_ArrayCopy();
+ if (ac->is_clonebasic()) {
+ intptr_t offset;
+ AllocateNode* alloc = AllocateNode::Ideal_allocation(ac->in(ArrayCopyNode::Dest), phase, offset);
+ assert(alloc != NULL && alloc->initialization()->is_complete_with_arraycopy(), "broken allocation");
+ if (alloc == ld_alloc) {
+ return ac;
+ }
+ }
+ }
+ } else if (mem->is_Proj() && mem->in(0) != NULL && mem->in(0)->is_ArrayCopy()) {
+ ArrayCopyNode* ac = mem->in(0)->as_ArrayCopy();
+
+ if (ac->is_arraycopy_validated() ||
+ ac->is_copyof_validated() ||
+ ac->is_copyofrange_validated()) {
+ Node* ld_addp = in(MemNode::Address);
+ if (ld_addp->is_AddP()) {
+ Node* ld_base = ld_addp->in(AddPNode::Address);
+ Node* ld_offs = ld_addp->in(AddPNode::Offset);
+
+ Node* dest = ac->in(ArrayCopyNode::Dest);
+
+ if (dest == ld_base) {
+ Node* src_pos = ac->in(ArrayCopyNode::SrcPos);
+ Node* dest_pos = ac->in(ArrayCopyNode::DestPos);
+ Node* len = ac->in(ArrayCopyNode::Length);
+
+ const TypeInt *dest_pos_t = phase->type(dest_pos)->isa_int();
+ const TypeX *ld_offs_t = phase->type(ld_offs)->isa_intptr_t();
+ const TypeInt *len_t = phase->type(len)->isa_int();
+ const TypeAryPtr* ary_t = phase->type(dest)->isa_aryptr();
+
+ if (dest_pos_t != NULL && ld_offs_t != NULL && len_t != NULL && ary_t != NULL) {
+ BasicType ary_elem = ary_t->klass()->as_array_klass()->element_type()->basic_type();
+ uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
+ uint elemsize = type2aelembytes(ary_elem);
+
+ intptr_t dest_pos_plus_len_lo = (((intptr_t)dest_pos_t->_lo) + len_t->_lo) * elemsize + header;
+ intptr_t dest_pos_plus_len_hi = (((intptr_t)dest_pos_t->_hi) + len_t->_hi) * elemsize + header;
+ intptr_t dest_pos_lo = ((intptr_t)dest_pos_t->_lo) * elemsize + header;
+ intptr_t dest_pos_hi = ((intptr_t)dest_pos_t->_hi) * elemsize + header;
+
+ if (can_see_stored_value) {
+ if (ld_offs_t->_lo >= dest_pos_hi && ld_offs_t->_hi < dest_pos_plus_len_lo) {
+ return ac;
+ }
+ } else {
+ if (ld_offs_t->_hi < dest_pos_lo || ld_offs_t->_lo >= dest_pos_plus_len_hi) {
+ mem = ac->in(TypeFunc::Memory);
+ }
+ return ac;
+ }
+ }
+ }
+ }
+ }
+ }
+ return NULL;
+}
+
// The logic for reordering loads and stores uses four steps:
// (a) Walk carefully past stores and initializations which we
// can prove are independent of this load.
@@ -510,6 +610,7 @@
for (;;) { // While we can dance past unrelated stores...
if (--cnt < 0) break; // Caught in cycle or a complicated dance?
+ Node* prev = mem;
if (mem->is_Store()) {
Node* st_adr = mem->in(MemNode::Address);
intptr_t st_offset = 0;
@@ -580,15 +681,26 @@
return mem; // let caller handle steps (c), (d)
}
+ } else if (find_previous_arraycopy(phase, alloc, mem, false) != NULL) {
+ if (prev != mem) {
+ // Found an arraycopy but it doesn't affect that load
+ continue;
+ }
+ // Found an arraycopy that may affect that load
+ return mem;
} else if (addr_t != NULL && addr_t->is_known_instance_field()) {
// Can't use optimize_simple_memory_chain() since it needs PhaseGVN.
if (mem->is_Proj() && mem->in(0)->is_Call()) {
+ // ArrayCopyNodes processed here as well.
CallNode *call = mem->in(0)->as_Call();
if (!call->may_modify(addr_t, phase)) {
mem = call->in(TypeFunc::Memory);
continue; // (a) advance through independent call memory
}
} else if (mem->is_Proj() && mem->in(0)->is_MemBar()) {
+ if (membar_for_arraycopy(addr_t, mem->in(0)->as_MemBar(), phase)) {
+ break;
+ }
mem = mem->in(0)->in(TypeFunc::Memory);
continue; // (a) advance through independent MemBar memory
} else if (mem->is_ClearArray()) {
@@ -760,6 +872,66 @@
return false;
}
+// Is the value loaded previously stored by an arraycopy? If so return
+// a load node that reads from the source array so we may be able to
+// optimize out the ArrayCopy node later.
+Node* MemNode::can_see_arraycopy_value(Node* st, PhaseTransform* phase) const {
+ Node* ld_adr = in(MemNode::Address);
+ intptr_t ld_off = 0;
+ AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
+ Node* ac = find_previous_arraycopy(phase, ld_alloc, st, true);
+ if (ac != NULL) {
+ assert(ac->is_ArrayCopy(), "what kind of node can this be?");
+ assert(is_Load(), "only for loads");
+
+ if (ac->as_ArrayCopy()->is_clonebasic()) {
+ assert(ld_alloc != NULL, "need an alloc");
+ Node* ld = clone();
+ Node* addp = in(MemNode::Address)->clone();
+ assert(addp->is_AddP(), "address must be addp");
+ assert(addp->in(AddPNode::Base) == ac->in(ArrayCopyNode::Dest)->in(AddPNode::Base), "strange pattern");
+ assert(addp->in(AddPNode::Address) == ac->in(ArrayCopyNode::Dest)->in(AddPNode::Address), "strange pattern");
+ addp->set_req(AddPNode::Base, ac->in(ArrayCopyNode::Src)->in(AddPNode::Base));
+ addp->set_req(AddPNode::Address, ac->in(ArrayCopyNode::Src)->in(AddPNode::Address));
+ ld->set_req(MemNode::Address, phase->transform(addp));
+ if (in(0) != NULL) {
+ assert(ld_alloc->in(0) != NULL, "alloc must have control");
+ ld->set_req(0, ld_alloc->in(0));
+ }
+ return ld;
+ } else {
+ Node* ld = clone();
+ Node* addp = in(MemNode::Address)->clone();
+ assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
+ addp->set_req(AddPNode::Base, ac->in(ArrayCopyNode::Src));
+ addp->set_req(AddPNode::Address, ac->in(ArrayCopyNode::Src));
+
+ const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
+ BasicType ary_elem = ary_t->klass()->as_array_klass()->element_type()->basic_type();
+ uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
+ uint shift = exact_log2(type2aelembytes(ary_elem));
+
+ Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
+#ifdef _LP64
+ diff = phase->transform(new ConvI2LNode(diff));
+#endif
+ diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
+
+ Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
+ addp->set_req(AddPNode::Offset, offset);
+ ld->set_req(MemNode::Address, phase->transform(addp));
+
+ if (in(0) != NULL) {
+ assert(ac->in(0) != NULL, "alloc must have control");
+ ld->set_req(0, ac->in(0));
+ }
+ return ld;
+ }
+ }
+ return NULL;
+}
+
+
//---------------------------can_see_stored_value------------------------------
// This routine exists to make sure this set of tests is done the same
// everywhere. We need to make a coordinated change: first LoadNode::Ideal
@@ -793,6 +965,7 @@
opc == Op_MemBarRelease ||
opc == Op_StoreFence ||
opc == Op_MemBarReleaseLock ||
+ opc == Op_MemBarStoreStore ||
opc == Op_MemBarCPUOrder) {
Node* mem = current->in(0)->in(TypeFunc::Memory);
if (mem->is_MergeMem()) {
@@ -863,8 +1036,9 @@
if ((alloc != NULL) && (alloc == ld_alloc)) {
// examine a captured store value
st = init->find_captured_store(ld_off, memory_size(), phase);
- if (st != NULL)
+ if (st != NULL) {
continue; // take one more trip around
+ }
}
}
@@ -1335,6 +1509,29 @@
}
}
+ // Is there a dominating load that loads the same value? Leave
+ // anything that is not a load of a field/array element (like
+ // barriers etc.) alone
+ if (in(0) != NULL && adr_type() != TypeRawPtr::BOTTOM && can_reshape) {
+ for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) {
+ Node *use = mem->fast_out(i);
+ if (use != this &&
+ use->Opcode() == Opcode() &&
+ use->in(0) != NULL &&
+ use->in(0) != in(0) &&
+ use->in(Address) == in(Address)) {
+ Node* ctl = in(0);
+ for (int i = 0; i < 10 && ctl != NULL; i++) {
+ ctl = IfNode::up_one_dom(ctl);
+ if (ctl == use->in(0)) {
+ set_req(0, use->in(0));
+ return this;
+ }
+ }
+ }
+ }
+ }
+
// Check for prior store with a different base or offset; make Load
// independent. Skip through any number of them. Bail out if the stores
// are in an endless dead cycle and report no progress. This is a key
@@ -1348,6 +1545,12 @@
// the alias index stuff. So instead, peek through Stores and IFF we can
// fold up, do so.
Node* prev_mem = find_previous_store(phase);
+ if (prev_mem != NULL) {
+ Node* value = can_see_arraycopy_value(prev_mem, phase);
+ if (value != NULL) {
+ return value;
+ }
+ }
// Steps (a), (b): Walk past independent stores to find an exact match.
if (prev_mem != NULL && prev_mem != in(MemNode::Memory)) {
// (c) See if we can fold up on the spot, but don't fold up here.
@@ -2529,7 +2732,6 @@
//=============================================================================
//-------------------------------adr_type--------------------------------------
-// Do we Match on this edge index or not? Do not match memory
const TypePtr* ClearArrayNode::adr_type() const {
Node *adr = in(3);
if (adr == NULL) return NULL; // node is dead