jdk-sandbox: comparison hotspot/src/share/vm/opto/memnode.cpp

equal deleted inserted replaced

-:3c15b4a3bf4d
+:b6e5ad2f18d5
 #include "classfile/systemDictionary.hpp"
 #include "compiler/compileLog.hpp"
 #include "memory/allocation.inline.hpp"
 #include "oops/objArrayKlass.hpp"
 #include "opto/addnode.hpp"
+#include "opto/arraycopynode.hpp"
 #include "opto/cfgnode.hpp"
 #include "opto/compile.hpp"
 #include "opto/connode.hpp"
 #include "opto/convertnode.hpp"
 #include "opto/loopnode.hpp"
 extern void print_alias_types();
 #endif
+static bool membar_for_arraycopy_helper(const TypeOopPtr *t_oop, MergeMemNode* mm, PhaseTransform *phase) {
+if (mm->memory_at(Compile::AliasIdxRaw)->is_Proj()) {
+Node* n = mm->memory_at(Compile::AliasIdxRaw)->in(0);
+if ((n->is_ArrayCopy() && n->as_ArrayCopy()->may_modify(t_oop, phase)) ||
+(n->is_CallLeaf() && n->as_CallLeaf()->may_modify(t_oop, phase))) {
+return true;
+}
+}
+return false;
+}
+static bool membar_for_arraycopy(const TypeOopPtr *t_oop, MemBarNode* mb, PhaseTransform *phase) {
+Node* mem = mb->in(TypeFunc::Memory);
+if (mem->is_MergeMem()) {
+return membar_for_arraycopy_helper(t_oop, mem->as_MergeMem(), phase);
+} else if (mem->is_Phi()) {
+// after macro expansion of an ArrayCopyNode we may have a Phi
+for (uint i = 1; i < mem->req(); i++) {
+if (mem->in(i) != NULL && mem->in(i)->is_MergeMem() && membar_for_arraycopy_helper(t_oop, mem->in(i)->as_MergeMem(), phase)) {
+return true;
+}
+}
+}
+return false;
+}
 Node *MemNode::optimize_simple_memory_chain(Node *mchain, const TypeOopPtr *t_oop, Node *load, PhaseGVN *phase) {
 assert((t_oop != NULL), "sanity");
 bool is_instance = t_oop->is_known_instance_field();
 bool is_boxed_value_load = t_oop->is_ptr_to_boxed_value() &&
 (load != NULL) && load->is_Load() &&
 if (result->is_Proj() && result->as_Proj()->_con == TypeFunc::Memory) {
 Node *proj_in = result->in(0);
 if (proj_in->is_Allocate() && proj_in->_idx == instance_id) {
 break;  // hit one of our sentinels
 } else if (proj_in->is_Call()) {
+// ArrayCopyNodes processed here as well
 CallNode *call = proj_in->as_Call();
 if (!call->may_modify(t_oop, phase)) { // returns false for instances
 result = call->in(TypeFunc::Memory);
 }
 } else if (proj_in->is_Initialize()) {
 AllocateNode* alloc = proj_in->as_Initialize()->allocation();
 // Stop if this is the initialization for the object instance which
-// which contains this memory slice, otherwise skip over it.
+// contains this memory slice, otherwise skip over it.
 if ((alloc == NULL) || (alloc->_idx == instance_id)) {
 break;
 }
 if (is_instance) {
 result = proj_in->in(TypeFunc::Memory);
 if (tklass->klass_is_exact() && !tklass->klass()->equals(t_oop->klass())) {
 result = proj_in->in(TypeFunc::Memory); // not related allocation
 }
 }
 } else if (proj_in->is_MemBar()) {
+if (membar_for_arraycopy(t_oop, proj_in->as_MemBar(), phase)) {
+break;
+}
 result = proj_in->in(TypeFunc::Memory);
 } else {
 assert(false, "unexpected projection");
 }
 } else if (result->is_ClearArray()) {
 }
 return false;
 }
+// Find an arraycopy that must have set (can_see_stored_value=true) or
+// could have set (can_see_stored_value=false) the value for this load
+Node* LoadNode::find_previous_arraycopy(PhaseTransform* phase, Node* ld_alloc, Node*& mem, bool can_see_stored_value) const {
+if (mem->is_Proj() && mem->in(0) != NULL && (mem->in(0)->Opcode() == Op_MemBarStoreStore ||
+mem->in(0)->Opcode() == Op_MemBarCPUOrder)) {
+Node* mb = mem->in(0);
+if (mb->in(0) != NULL && mb->in(0)->is_Proj() &&
+mb->in(0)->in(0) != NULL && mb->in(0)->in(0)->is_ArrayCopy()) {
+ArrayCopyNode* ac = mb->in(0)->in(0)->as_ArrayCopy();
+if (ac->is_clonebasic()) {
+intptr_t offset;
+AllocateNode* alloc = AllocateNode::Ideal_allocation(ac->in(ArrayCopyNode::Dest), phase, offset);
+assert(alloc != NULL && alloc->initialization()->is_complete_with_arraycopy(), "broken allocation");
+if (alloc == ld_alloc) {
+return ac;
+}
+}
+}
+} else if (mem->is_Proj() && mem->in(0) != NULL && mem->in(0)->is_ArrayCopy()) {
+ArrayCopyNode* ac = mem->in(0)->as_ArrayCopy();
+if (ac->is_arraycopy_validated() ||
+ac->is_copyof_validated() ||
+ac->is_copyofrange_validated()) {
+Node* ld_addp = in(MemNode::Address);
+if (ld_addp->is_AddP()) {
+Node* ld_base = ld_addp->in(AddPNode::Address);
+Node* ld_offs = ld_addp->in(AddPNode::Offset);
+Node* dest = ac->in(ArrayCopyNode::Dest);
+if (dest == ld_base) {
+Node* src_pos = ac->in(ArrayCopyNode::SrcPos);
+Node* dest_pos = ac->in(ArrayCopyNode::DestPos);
+Node* len = ac->in(ArrayCopyNode::Length);
+const TypeInt *dest_pos_t = phase->type(dest_pos)->isa_int();
+const TypeX *ld_offs_t = phase->type(ld_offs)->isa_intptr_t();
+const TypeInt *len_t = phase->type(len)->isa_int();
+const TypeAryPtr* ary_t = phase->type(dest)->isa_aryptr();
+if (dest_pos_t != NULL && ld_offs_t != NULL && len_t != NULL && ary_t != NULL) {
+BasicType ary_elem  = ary_t->klass()->as_array_klass()->element_type()->basic_type();
+uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
+uint elemsize = type2aelembytes(ary_elem);
+intptr_t dest_pos_plus_len_lo = (((intptr_t)dest_pos_t->_lo) + len_t->_lo) * elemsize + header;
+intptr_t dest_pos_plus_len_hi = (((intptr_t)dest_pos_t->_hi) + len_t->_hi) * elemsize + header;
+intptr_t dest_pos_lo = ((intptr_t)dest_pos_t->_lo) * elemsize + header;
+intptr_t dest_pos_hi = ((intptr_t)dest_pos_t->_hi) * elemsize + header;
+if (can_see_stored_value) {
+if (ld_offs_t->_lo >= dest_pos_hi && ld_offs_t->_hi < dest_pos_plus_len_lo) {
+return ac;
+}
+} else {
+if (ld_offs_t->_hi < dest_pos_lo || ld_offs_t->_lo >= dest_pos_plus_len_hi) {
+mem = ac->in(TypeFunc::Memory);
+}
+return ac;
+}
+}
+}
+}
+}
+}
+return NULL;
+}
 // The logic for reordering loads and stores uses four steps:
 // (a) Walk carefully past stores and initializations which we
 //     can prove are independent of this load.
 // (b) Observe that the next memory state makes an exact match
 //     with self (load or store), and locate the relevant store.
 int cnt = 50;             // Cycle limiter
 for (;;) {                // While we can dance past unrelated stores...
 if (--cnt < 0)  break;  // Caught in cycle or a complicated dance?
+Node* prev = mem;
 if (mem->is_Store()) {
 Node* st_adr = mem->in(MemNode::Address);
 intptr_t st_offset = 0;
 Node* st_base = AddPNode::Ideal_base_and_offset(st_adr, phase, st_offset);
 if (st_base == NULL)
 if (known_identical) {
 // From caller, can_see_stored_value will consult find_captured_store.
 return mem;         // let caller handle steps (c), (d)
 }
+} else if (find_previous_arraycopy(phase, alloc, mem, false) != NULL) {
+if (prev != mem) {
+// Found an arraycopy but it doesn't affect that load
+continue;
+}
+// Found an arraycopy that may affect that load
+return mem;
 } else if (addr_t != NULL && addr_t->is_known_instance_field()) {
 // Can't use optimize_simple_memory_chain() since it needs PhaseGVN.
 if (mem->is_Proj() && mem->in(0)->is_Call()) {
+// ArrayCopyNodes processed here as well.
 CallNode *call = mem->in(0)->as_Call();
 if (!call->may_modify(addr_t, phase)) {
 mem = call->in(TypeFunc::Memory);
 continue;         // (a) advance through independent call memory
 }
 } else if (mem->is_Proj() && mem->in(0)->is_MemBar()) {
+if (membar_for_arraycopy(addr_t, mem->in(0)->as_MemBar(), phase)) {
+break;
+}
 mem = mem->in(0)->in(TypeFunc::Memory);
 continue;           // (a) advance through independent MemBar memory
 } else if (mem->is_ClearArray()) {
 if (ClearArrayNode::step_through(&mem, (uint)addr_t->instance_id(), phase)) {
 // (the call updated 'mem' value)
 return (eliminate_boxing && non_volatile) || is_stable_ary;
 }
 return false;
 }
+// Is the value loaded previously stored by an arraycopy? If so return
+// a load node that reads from the source array so we may be able to
+// optimize out the ArrayCopy node later.
+Node* MemNode::can_see_arraycopy_value(Node* st, PhaseTransform* phase) const {
+Node* ld_adr = in(MemNode::Address);
+intptr_t ld_off = 0;
+AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
+Node* ac = find_previous_arraycopy(phase, ld_alloc, st, true);
+if (ac != NULL) {
+assert(ac->is_ArrayCopy(), "what kind of node can this be?");
+assert(is_Load(), "only for loads");
+if (ac->as_ArrayCopy()->is_clonebasic()) {
+assert(ld_alloc != NULL, "need an alloc");
+Node* ld = clone();
+Node* addp = in(MemNode::Address)->clone();
+assert(addp->is_AddP(), "address must be addp");
+assert(addp->in(AddPNode::Base) == ac->in(ArrayCopyNode::Dest)->in(AddPNode::Base), "strange pattern");
+assert(addp->in(AddPNode::Address) == ac->in(ArrayCopyNode::Dest)->in(AddPNode::Address), "strange pattern");
+addp->set_req(AddPNode::Base, ac->in(ArrayCopyNode::Src)->in(AddPNode::Base));
+addp->set_req(AddPNode::Address, ac->in(ArrayCopyNode::Src)->in(AddPNode::Address));
+ld->set_req(MemNode::Address, phase->transform(addp));
+if (in(0) != NULL) {
+assert(ld_alloc->in(0) != NULL, "alloc must have control");
+ld->set_req(0, ld_alloc->in(0));
+}
+return ld;
+} else {
+Node* ld = clone();
+Node* addp = in(MemNode::Address)->clone();
+assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
+addp->set_req(AddPNode::Base, ac->in(ArrayCopyNode::Src));
+addp->set_req(AddPNode::Address, ac->in(ArrayCopyNode::Src));
+const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
+BasicType ary_elem  = ary_t->klass()->as_array_klass()->element_type()->basic_type();
+uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
+uint shift  = exact_log2(type2aelembytes(ary_elem));
+Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
+#ifdef _LP64
+diff = phase->transform(new ConvI2LNode(diff));
+#endif
+diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
+Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
+addp->set_req(AddPNode::Offset, offset);
+ld->set_req(MemNode::Address, phase->transform(addp));
+if (in(0) != NULL) {
+assert(ac->in(0) != NULL, "alloc must have control");
+ld->set_req(0, ac->in(0));
+}
+return ld;
+}
+}
+return NULL;
+}
 //---------------------------can_see_stored_value------------------------------
 // This routine exists to make sure this set of tests is done the same
 // everywhere.  We need to make a coordinated change: first LoadNode::Ideal
 // will change the graph shape in a way which makes memory alive twice at the
 opc == Op_MemBarAcquireLock ||
 opc == Op_LoadFence)) ||
 opc == Op_MemBarRelease ||
 opc == Op_StoreFence ||
 opc == Op_MemBarReleaseLock ||
+opc == Op_MemBarStoreStore ||
 opc == Op_MemBarCPUOrder) {
 Node* mem = current->in(0)->in(TypeFunc::Memory);
 if (mem->is_MergeMem()) {
 MergeMemNode* merge = mem->as_MergeMem();
 Node* new_st = merge->memory_at(alias_idx);
 InitializeNode* init = st->in(0)->as_Initialize();
 AllocateNode* alloc = init->allocation();
 if ((alloc != NULL) && (alloc == ld_alloc)) {
 // examine a captured store value
 st = init->find_captured_store(ld_off, memory_size(), phase);
-if (st != NULL)
+if (st != NULL) {
 continue;             // take one more trip around
+}
 }
 }
 // Load boxed value from result of valueOf() call is input parameter.
 if (this->is_Load() && ld_adr->is_AddP() &&
 if (result != NULL) return result;
 }
 }
 }
+// Is there a dominating load that loads the same value?  Leave
+// anything that is not a load of a field/array element (like
+// barriers etc.) alone
+if (in(0) != NULL && adr_type() != TypeRawPtr::BOTTOM && can_reshape) {
+for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) {
+Node *use = mem->fast_out(i);
+if (use != this &&
+use->Opcode() == Opcode() &&
+use->in(0) != NULL &&
+use->in(0) != in(0) &&
+use->in(Address) == in(Address)) {
+Node* ctl = in(0);
+for (int i = 0; i < 10 && ctl != NULL; i++) {
+ctl = IfNode::up_one_dom(ctl);
+if (ctl == use->in(0)) {
+set_req(0, use->in(0));
+return this;
+}
+}
+}
+}
+}
 // Check for prior store with a different base or offset; make Load
 // independent.  Skip through any number of them.  Bail out if the stores
 // are in an endless dead cycle and report no progress.  This is a key
 // transform for Reflection.  However, if after skipping through the Stores
 // we can't then fold up against a prior store do NOT do the transform as
 // anti-dependence work knows how to bypass.  I.e. we need all
 // anti-dependence checks to ask the same Oracle.  Right now, that Oracle is
 // the alias index stuff.  So instead, peek through Stores and IFF we can
 // fold up, do so.
 Node* prev_mem = find_previous_store(phase);
+if (prev_mem != NULL) {
+Node* value = can_see_arraycopy_value(prev_mem, phase);
+if (value != NULL) {
+return value;
+}
+}
 // Steps (a), (b):  Walk past independent stores to find an exact match.
 if (prev_mem != NULL && prev_mem != in(MemNode::Memory)) {
 // (c) See if we can fold up on the spot, but don't fold up here.
 // Fold-up might require truncation (for LoadB/LoadS/LoadUS) or
 // just return a prior value, which is done by Identity calls.
 init_req(ExpectedIn, ex );
 }
 //=============================================================================
 //-------------------------------adr_type--------------------------------------
-// Do we Match on this edge index or not?  Do not match memory
 const TypePtr* ClearArrayNode::adr_type() const {
 Node *adr = in(3);
 if (adr == NULL)  return NULL; // node is dead
 return MemNode::calculate_adr_type(adr->bottom_type());
 }

changeset 30629	b6e5ad2f18d5
parent 30300	4b12a5b40064
child 31035	0f0743952c41