1 /*

     2  * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.

     3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

     4  *

     5  * This code is free software; you can redistribute it and/or modify it

     6  * under the terms of the GNU General Public License version 2 only, as

     7  * published by the Free Software Foundation.

     8  *

     9  * This code is distributed in the hope that it will be useful, but WITHOUT

    10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    12  * version 2 for more details (a copy is included in the LICENSE file that

    13  * accompanied this code).

    14  *

    15  * You should have received a copy of the GNU General Public License version

    16  * 2 along with this work; if not, write to the Free Software Foundation,

    17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    18  *

    19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

    20  * or visit www.oracle.com if you need additional information or have any

    21  * questions.

    22  *

    23  */

    24 

    25 #include "precompiled.hpp"

    26 #include "opto/arraycopynode.hpp"

    27 #include "opto/graphKit.hpp"

    28 

    29 ArrayCopyNode::ArrayCopyNode(Compile* C, bool alloc_tightly_coupled)

    30   : CallNode(arraycopy_type(), NULL, TypeRawPtr::BOTTOM),

    31     _alloc_tightly_coupled(alloc_tightly_coupled),

    32     _kind(None),

    33     _arguments_validated(false) {

    34   init_class_id(Class_ArrayCopy);

    35   init_flags(Flag_is_macro);

    36   C->add_macro_node(this);

    37 }

    38 

    39 uint ArrayCopyNode::size_of() const { return sizeof(*this); }

    40 

    41 ArrayCopyNode* ArrayCopyNode::make(GraphKit* kit, bool may_throw,

    42                                    Node* src, Node* src_offset,

    43                                    Node* dest, Node* dest_offset,

    44                                    Node* length,

    45                                    bool alloc_tightly_coupled,

    46                                    Node* src_klass, Node* dest_klass,

    47                                    Node* src_length, Node* dest_length) {

    48 

    49   ArrayCopyNode* ac = new ArrayCopyNode(kit->C, alloc_tightly_coupled);

    50   Node* prev_mem = kit->set_predefined_input_for_runtime_call(ac);

    51 

    52   ac->init_req(ArrayCopyNode::Src, src);

    53   ac->init_req(ArrayCopyNode::SrcPos, src_offset);

    54   ac->init_req(ArrayCopyNode::Dest, dest);

    55   ac->init_req(ArrayCopyNode::DestPos, dest_offset);

    56   ac->init_req(ArrayCopyNode::Length, length);

    57   ac->init_req(ArrayCopyNode::SrcLen, src_length);

    58   ac->init_req(ArrayCopyNode::DestLen, dest_length);

    59   ac->init_req(ArrayCopyNode::SrcKlass, src_klass);

    60   ac->init_req(ArrayCopyNode::DestKlass, dest_klass);

    61 

    62   if (may_throw) {

    63     ac->set_req(TypeFunc::I_O , kit->i_o());

    64     kit->add_safepoint_edges(ac, false);

    65   }

    66 

    67   return ac;

    68 }

    69 

    70 void ArrayCopyNode::connect_outputs(GraphKit* kit) {

    71   kit->set_all_memory_call(this, true);

    72   kit->set_control(kit->gvn().transform(new ProjNode(this,TypeFunc::Control)));

    73   kit->set_i_o(kit->gvn().transform(new ProjNode(this, TypeFunc::I_O)));

    74   kit->make_slow_call_ex(this, kit->env()->Throwable_klass(), true);

    75   kit->set_all_memory_call(this);

    76 }

    77 

    78 #ifndef PRODUCT

    79 const char* ArrayCopyNode::_kind_names[] = {"arraycopy", "arraycopy, validated arguments", "clone", "oop array clone", "CopyOf", "CopyOfRange"};

    80 void ArrayCopyNode::dump_spec(outputStream *st) const {

    81   CallNode::dump_spec(st);

    82   st->print(" (%s%s)", _kind_names[_kind], _alloc_tightly_coupled ? ", tightly coupled allocation" : "");

    83 }

    84 #endif

    85 

    86 intptr_t ArrayCopyNode::get_length_if_constant(PhaseGVN *phase) const {

    87   // check that length is constant

    88   Node* length = in(ArrayCopyNode::Length);

    89   const Type* length_type = phase->type(length);

    90 

    91   if (length_type == Type::TOP) {

    92     return -1;

    93   }

    94 

    95   assert(is_clonebasic() || is_arraycopy() || is_copyof() || is_copyofrange(), "unexpected array copy type");

    96 

    97   return is_clonebasic() ? length->find_intptr_t_con(-1) : length->find_int_con(-1);

    98 }

    99 

   100 int ArrayCopyNode::get_count(PhaseGVN *phase) const {

   101   Node* src = in(ArrayCopyNode::Src);

   102   const Type* src_type = phase->type(src);

   103 

   104   if (is_clonebasic()) {

   105     if (src_type->isa_instptr()) {

   106       const TypeInstPtr* inst_src = src_type->is_instptr();

   107       ciInstanceKlass* ik = inst_src->klass()->as_instance_klass();

   108       // ciInstanceKlass::nof_nonstatic_fields() doesn't take injected

   109       // fields into account. They are rare anyway so easier to simply

   110       // skip instances with injected fields.

   111       if ((!inst_src->klass_is_exact() && (ik->is_interface() || ik->has_subklass())) || ik->has_injected_fields()) {

   112         return -1;

   113       }

   114       int nb_fields = ik->nof_nonstatic_fields();

   115       return nb_fields;

   116     } else {

   117       const TypeAryPtr* ary_src = src_type->isa_aryptr();

   118       assert (ary_src != NULL, "not an array or instance?");

   119       // clone passes a length as a rounded number of longs. If we're

   120       // cloning an array we'll do it element by element. If the

   121       // length input to ArrayCopyNode is constant, length of input

   122       // array must be too.

   123 

   124       assert((get_length_if_constant(phase) == -1) == !ary_src->size()->is_con(), "inconsistent");

   125 

   126       if (ary_src->size()->is_con()) {

   127         return ary_src->size()->get_con();

   128       }

   129       return -1;

   130     }

   131   }

   132 

   133   return get_length_if_constant(phase);

   134 }

   135 

   136 Node* ArrayCopyNode::try_clone_instance(PhaseGVN *phase, bool can_reshape, int count) {

   137   if (!is_clonebasic()) {

   138     return NULL;

   139   }

   140 

   141   Node* src = in(ArrayCopyNode::Src);

   142   Node* dest = in(ArrayCopyNode::Dest);

   143   Node* ctl = in(TypeFunc::Control);

   144   Node* in_mem = in(TypeFunc::Memory);

   145 

   146   const Type* src_type = phase->type(src);

   147   const Type* dest_type = phase->type(dest);

   148 

   149   assert(src->is_AddP(), "should be base + off");

   150   assert(dest->is_AddP(), "should be base + off");

   151   Node* base_src = src->in(AddPNode::Base);

   152   Node* base_dest = dest->in(AddPNode::Base);

   153 

   154   MergeMemNode* mem = MergeMemNode::make(in_mem);

   155 

   156   const TypeInstPtr* inst_src = src_type->isa_instptr();

   157 

   158   if (inst_src == NULL) {

   159     return NULL;

   160   }

   161 

   162   if (!inst_src->klass_is_exact()) {

   163     ciInstanceKlass* ik = inst_src->klass()->as_instance_klass();

   164     assert(!ik->is_interface() && !ik->has_subklass(), "inconsistent klass hierarchy");

   165     phase->C->dependencies()->assert_leaf_type(ik);

   166   }

   167 

   168   ciInstanceKlass* ik = inst_src->klass()->as_instance_klass();

   169   assert(ik->nof_nonstatic_fields() <= ArrayCopyLoadStoreMaxElem, "too many fields");

   170 

   171   for (int i = 0; i < count; i++) {

   172     ciField* field = ik->nonstatic_field_at(i);

   173     int fieldidx = phase->C->alias_type(field)->index();

   174     const TypePtr* adr_type = phase->C->alias_type(field)->adr_type();

   175     Node* off = phase->MakeConX(field->offset());

   176     Node* next_src = phase->transform(new AddPNode(base_src,base_src,off));

   177     Node* next_dest = phase->transform(new AddPNode(base_dest,base_dest,off));

   178     BasicType bt = field->layout_type();

   179 

   180     const Type *type;

   181     if (bt == T_OBJECT) {

   182       if (!field->type()->is_loaded()) {

   183         type = TypeInstPtr::BOTTOM;

   184       } else {

   185         ciType* field_klass = field->type();

   186         type = TypeOopPtr::make_from_klass(field_klass->as_klass());

   187       }

   188     } else {

   189       type = Type::get_const_basic_type(bt);

   190     }

   191 

   192     Node* v = LoadNode::make(*phase, ctl, mem->memory_at(fieldidx), next_src, adr_type, type, bt, MemNode::unordered);

   193     v = phase->transform(v);

   194     Node* s = StoreNode::make(*phase, ctl, mem->memory_at(fieldidx), next_dest, adr_type, v, bt, MemNode::unordered);

   195     s = phase->transform(s);

   196     mem->set_memory_at(fieldidx, s);

   197   }

   198 

   199   if (!finish_transform(phase, can_reshape, ctl, mem)) {

   200     return NULL;

   201   }

   202 

   203   return mem;

   204 }

   205 

   206 bool ArrayCopyNode::prepare_array_copy(PhaseGVN *phase, bool can_reshape,

   207                                        Node*& adr_src,

   208                                        Node*& base_src,

   209                                        Node*& adr_dest,

   210                                        Node*& base_dest,

   211                                        BasicType& copy_type,

   212                                        const Type*& value_type,

   213                                        bool& disjoint_bases) {

   214   Node* src = in(ArrayCopyNode::Src);

   215   Node* dest = in(ArrayCopyNode::Dest);

   216   const Type* src_type = phase->type(src);

   217   const TypeAryPtr* ary_src = src_type->isa_aryptr();

   218 

   219   if (is_arraycopy() || is_copyofrange() || is_copyof()) {

   220     const Type* dest_type = phase->type(dest);

   221     const TypeAryPtr* ary_dest = dest_type->isa_aryptr();

   222     Node* src_offset = in(ArrayCopyNode::SrcPos);

   223     Node* dest_offset = in(ArrayCopyNode::DestPos);

   224 

   225     // newly allocated object is guaranteed to not overlap with source object

   226     disjoint_bases = is_alloc_tightly_coupled();

   227 

   228     if (ary_src  == NULL || ary_src->klass()  == NULL ||

   229         ary_dest == NULL || ary_dest->klass() == NULL) {

   230       // We don't know if arguments are arrays

   231       return false;

   232     }

   233 

   234     BasicType src_elem  = ary_src->klass()->as_array_klass()->element_type()->basic_type();

   235     BasicType dest_elem = ary_dest->klass()->as_array_klass()->element_type()->basic_type();

   236     if (src_elem  == T_ARRAY)  src_elem  = T_OBJECT;

   237     if (dest_elem == T_ARRAY)  dest_elem = T_OBJECT;

   238 

   239     if (src_elem != dest_elem || dest_elem == T_VOID) {

   240       // We don't know if arguments are arrays of the same type

   241       return false;

   242     }

   243 

   244     if (dest_elem == T_OBJECT && (!is_alloc_tightly_coupled() || !GraphKit::use_ReduceInitialCardMarks())) {

   245       // It's an object array copy but we can't emit the card marking

   246       // that is needed

   247       return false;

   248     }

   249 

   250     value_type = ary_src->elem();

   251 

   252     base_src = src;

   253     base_dest = dest;

   254 

   255     uint shift  = exact_log2(type2aelembytes(dest_elem));

   256     uint header = arrayOopDesc::base_offset_in_bytes(dest_elem);

   257 

   258     adr_src = src;

   259     adr_dest = dest;

   260 

   261     src_offset = Compile::conv_I2X_index(phase, src_offset, ary_src->size());

   262     dest_offset = Compile::conv_I2X_index(phase, dest_offset, ary_dest->size());

   263 

   264     Node* src_scale = phase->transform(new LShiftXNode(src_offset, phase->intcon(shift)));

   265     Node* dest_scale = phase->transform(new LShiftXNode(dest_offset, phase->intcon(shift)));

   266 

   267     adr_src = phase->transform(new AddPNode(base_src, adr_src, src_scale));

   268     adr_dest = phase->transform(new AddPNode(base_dest, adr_dest, dest_scale));

   269 

   270     adr_src = new AddPNode(base_src, adr_src, phase->MakeConX(header));

   271     adr_dest = new AddPNode(base_dest, adr_dest, phase->MakeConX(header));

   272 

   273     adr_src = phase->transform(adr_src);

   274     adr_dest = phase->transform(adr_dest);

   275 

   276     copy_type = dest_elem;

   277   } else {

   278     assert (is_clonebasic(), "should be");

   279 

   280     disjoint_bases = true;

   281     assert(src->is_AddP(), "should be base + off");

   282     assert(dest->is_AddP(), "should be base + off");

   283     adr_src = src;

   284     base_src = src->in(AddPNode::Base);

   285     adr_dest = dest;

   286     base_dest = dest->in(AddPNode::Base);

   287 

   288     assert(phase->type(src->in(AddPNode::Offset))->is_intptr_t()->get_con() == phase->type(dest->in(AddPNode::Offset))->is_intptr_t()->get_con(), "same start offset?");

   289     BasicType elem = ary_src->klass()->as_array_klass()->element_type()->basic_type();

   290     if (elem == T_ARRAY)  elem = T_OBJECT;

   291 

   292     int diff = arrayOopDesc::base_offset_in_bytes(elem) - phase->type(src->in(AddPNode::Offset))->is_intptr_t()->get_con();

   293     assert(diff >= 0, "clone should not start after 1st array element");

   294     if (diff > 0) {

   295       adr_src = phase->transform(new AddPNode(base_src, adr_src, phase->MakeConX(diff)));

   296       adr_dest = phase->transform(new AddPNode(base_dest, adr_dest, phase->MakeConX(diff)));

   297     }

   298 

   299     copy_type = elem;

   300     value_type = ary_src->elem();

   301   }

   302   return true;

   303 }

   304 

   305 const TypePtr* ArrayCopyNode::get_address_type(PhaseGVN *phase, Node* n) {

   306   const Type* at = phase->type(n);

   307   assert(at != Type::TOP, "unexpected type");

   308   const TypePtr* atp = at->isa_ptr();

   309   // adjust atp to be the correct array element address type

   310   atp = atp->add_offset(Type::OffsetBot);

   311   return atp;

   312 }

   313 

   314 void ArrayCopyNode::array_copy_test_overlap(PhaseGVN *phase, bool can_reshape, bool disjoint_bases, int count, Node*& forward_ctl, Node*& backward_ctl) {

   315   Node* ctl = in(TypeFunc::Control);

   316   if (!disjoint_bases && count > 1) {

   317     Node* src_offset = in(ArrayCopyNode::SrcPos);

   318     Node* dest_offset = in(ArrayCopyNode::DestPos);

   319     assert(src_offset != NULL && dest_offset != NULL, "should be");

   320     Node* cmp = phase->transform(new CmpINode(src_offset, dest_offset));

   321     Node *bol = phase->transform(new BoolNode(cmp, BoolTest::lt));

   322     IfNode *iff = new IfNode(ctl, bol, PROB_FAIR, COUNT_UNKNOWN);

   323 

   324     phase->transform(iff);

   325 

   326     forward_ctl = phase->transform(new IfFalseNode(iff));

   327     backward_ctl = phase->transform(new IfTrueNode(iff));

   328   } else {

   329     forward_ctl = ctl;

   330   }

   331 }

   332 

   333 Node* ArrayCopyNode::array_copy_forward(PhaseGVN *phase,

   334                                         bool can_reshape,

   335                                         Node* forward_ctl,

   336                                         Node* start_mem_src,

   337                                         Node* start_mem_dest,

   338                                         const TypePtr* atp_src,

   339                                         const TypePtr* atp_dest,

   340                                         Node* adr_src,

   341                                         Node* base_src,

   342                                         Node* adr_dest,

   343                                         Node* base_dest,

   344                                         BasicType copy_type,

   345                                         const Type* value_type,

   346                                         int count) {

   347   Node* mem = phase->C->top();

   348   if (!forward_ctl->is_top()) {

   349     // copy forward

   350     mem = start_mem_dest;

   351 

   352     if (count > 0) {

   353       Node* v = LoadNode::make(*phase, forward_ctl, start_mem_src, adr_src, atp_src, value_type, copy_type, MemNode::unordered);

   354       v = phase->transform(v);

   355       mem = StoreNode::make(*phase, forward_ctl, mem, adr_dest, atp_dest, v, copy_type, MemNode::unordered);

   356       mem = phase->transform(mem);

   357       for (int i = 1; i < count; i++) {

   358         Node* off  = phase->MakeConX(type2aelembytes(copy_type) * i);

   359         Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off));

   360         Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off));

   361         v = LoadNode::make(*phase, forward_ctl, mem, next_src, atp_src, value_type, copy_type, MemNode::unordered);

   362         v = phase->transform(v);

   363         mem = StoreNode::make(*phase, forward_ctl,mem,next_dest,atp_dest,v, copy_type, MemNode::unordered);

   364         mem = phase->transform(mem);

   365       }

   366     } else if(can_reshape) {

   367       PhaseIterGVN* igvn = phase->is_IterGVN();

   368       igvn->_worklist.push(adr_src);

   369       igvn->_worklist.push(adr_dest);

   370     }

   371   }

   372   return mem;

   373 }

   374 

   375 Node* ArrayCopyNode::array_copy_backward(PhaseGVN *phase,

   376                                          bool can_reshape,

   377                                          Node* backward_ctl,

   378                                          Node* start_mem_src,

   379                                          Node* start_mem_dest,

   380                                          const TypePtr* atp_src,

   381                                          const TypePtr* atp_dest,

   382                                          Node* adr_src,

   383                                          Node* base_src,

   384                                          Node* adr_dest,

   385                                          Node* base_dest,

   386                                          BasicType copy_type,

   387                                          const Type* value_type,

   388                                          int count) {

   389   Node* mem = phase->C->top();

   390   if (!backward_ctl->is_top()) {

   391     // copy backward

   392     mem = start_mem_dest;

   393 

   394     if (count > 0) {

   395       for (int i = count-1; i >= 1; i--) {

   396         Node* off  = phase->MakeConX(type2aelembytes(copy_type) * i);

   397         Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off));

   398         Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off));

   399         Node* v = LoadNode::make(*phase, backward_ctl, mem, next_src, atp_src, value_type, copy_type, MemNode::unordered);

   400         v = phase->transform(v);

   401         mem = StoreNode::make(*phase, backward_ctl,mem,next_dest,atp_dest,v, copy_type, MemNode::unordered);

   402         mem = phase->transform(mem);

   403       }

   404       Node* v = LoadNode::make(*phase, backward_ctl, mem, adr_src, atp_src, value_type, copy_type, MemNode::unordered);

   405       v = phase->transform(v);

   406       mem = StoreNode::make(*phase, backward_ctl, mem, adr_dest, atp_dest, v, copy_type, MemNode::unordered);

   407       mem = phase->transform(mem);

   408     } else if(can_reshape) {

   409       PhaseIterGVN* igvn = phase->is_IterGVN();

   410       igvn->_worklist.push(adr_src);

   411       igvn->_worklist.push(adr_dest);

   412     }

   413   }

   414   return mem;

   415 }

   416 

   417 bool ArrayCopyNode::finish_transform(PhaseGVN *phase, bool can_reshape,

   418                                      Node* ctl, Node *mem) {

   419   if (can_reshape) {

   420     PhaseIterGVN* igvn = phase->is_IterGVN();

   421     igvn->set_delay_transform(false);

   422     if (is_clonebasic()) {

   423       Node* out_mem = proj_out(TypeFunc::Memory);

   424 

   425       if (out_mem->outcnt() != 1 || !out_mem->raw_out(0)->is_MergeMem() ||

   426           out_mem->raw_out(0)->outcnt() != 1 || !out_mem->raw_out(0)->raw_out(0)->is_MemBar()) {

   427         assert(!GraphKit::use_ReduceInitialCardMarks(), "can only happen with card marking");

   428         return false;

   429       }

   430 

   431       igvn->replace_node(out_mem->raw_out(0), mem);

   432 

   433       Node* out_ctl = proj_out(TypeFunc::Control);

   434       igvn->replace_node(out_ctl, ctl);

   435     } else {

   436       // replace fallthrough projections of the ArrayCopyNode by the

   437       // new memory, control and the input IO.

   438       CallProjections callprojs;

   439       extract_projections(&callprojs, true);

   440 

   441       igvn->replace_node(callprojs.fallthrough_ioproj, in(TypeFunc::I_O));

   442       igvn->replace_node(callprojs.fallthrough_memproj, mem);

   443       igvn->replace_node(callprojs.fallthrough_catchproj, ctl);

   444 

   445       // The ArrayCopyNode is not disconnected. It still has the

   446       // projections for the exception case. Replace current

   447       // ArrayCopyNode with a dummy new one with a top() control so

   448       // that this part of the graph stays consistent but is

   449       // eventually removed.

   450 

   451       set_req(0, phase->C->top());

   452       remove_dead_region(phase, can_reshape);

   453     }

   454   } else {

   455     if (in(TypeFunc::Control) != ctl) {

   456       // we can't return new memory and control from Ideal at parse time

   457       assert(!is_clonebasic(), "added control for clone?");

   458       return NULL;

   459     }

   460   }

   461   return true;

   462 }

   463 

   464 

   465 Node *ArrayCopyNode::Ideal(PhaseGVN *phase, bool can_reshape) {

   466   if (remove_dead_region(phase, can_reshape))  return this;

   467 

   468   if (StressArrayCopyMacroNode && !can_reshape) {

   469     phase->record_for_igvn(this);

   470     return NULL;

   471   }

   472 

   473   // See if it's a small array copy and we can inline it as

   474   // loads/stores

   475   // Here we can only do:

   476   // - arraycopy if all arguments were validated before and we don't

   477   // need card marking

   478   // - clone for which we don't need to do card marking

   479 

   480   if (!is_clonebasic() && !is_arraycopy_validated() &&

   481       !is_copyofrange_validated() && !is_copyof_validated()) {

   482     return NULL;

   483   }

   484 

   485   if (in(TypeFunc::Control)->is_top() || in(TypeFunc::Memory)->is_top()) {

   486     return NULL;

   487   }

   488 

   489   int count = get_count(phase);

   490 

   491   if (count < 0 || count > ArrayCopyLoadStoreMaxElem) {

   492     return NULL;

   493   }

   494 

   495   Node* mem = try_clone_instance(phase, can_reshape, count);

   496   if (mem != NULL) {

   497     return mem;

   498   }

   499 

   500   Node* adr_src = NULL;

   501   Node* base_src = NULL;

   502   Node* adr_dest = NULL;

   503   Node* base_dest = NULL;

   504   BasicType copy_type = T_ILLEGAL;

   505   const Type* value_type = NULL;

   506   bool disjoint_bases = false;

   507 

   508   if (!prepare_array_copy(phase, can_reshape,

   509                           adr_src, base_src, adr_dest, base_dest,

   510                           copy_type, value_type, disjoint_bases)) {

   511     return NULL;

   512   }

   513 

   514   Node* src = in(ArrayCopyNode::Src);

   515   Node* dest = in(ArrayCopyNode::Dest);

   516   const TypePtr* atp_src = get_address_type(phase, src);

   517   const TypePtr* atp_dest = get_address_type(phase, dest);

   518   uint alias_idx_src = phase->C->get_alias_index(atp_src);

   519   uint alias_idx_dest = phase->C->get_alias_index(atp_dest);

   520 

   521   Node *in_mem = in(TypeFunc::Memory);

   522   Node *start_mem_src = in_mem;

   523   Node *start_mem_dest = in_mem;

   524   if (in_mem->is_MergeMem()) {

   525     start_mem_src = in_mem->as_MergeMem()->memory_at(alias_idx_src);

   526     start_mem_dest = in_mem->as_MergeMem()->memory_at(alias_idx_dest);

   527   }

   528 

   529 

   530   if (can_reshape) {

   531     assert(!phase->is_IterGVN()->delay_transform(), "cannot delay transforms");

   532     phase->is_IterGVN()->set_delay_transform(true);

   533   }

   534 

   535   Node* backward_ctl = phase->C->top();

   536   Node* forward_ctl = phase->C->top();

   537   array_copy_test_overlap(phase, can_reshape, disjoint_bases, count, forward_ctl, backward_ctl);

   538 

   539   Node* forward_mem = array_copy_forward(phase, can_reshape, forward_ctl,

   540                                          start_mem_src, start_mem_dest,

   541                                          atp_src, atp_dest,

   542                                          adr_src, base_src, adr_dest, base_dest,

   543                                          copy_type, value_type, count);

   544 

   545   Node* backward_mem = array_copy_backward(phase, can_reshape, backward_ctl,

   546                                            start_mem_src, start_mem_dest,

   547                                            atp_src, atp_dest,

   548                                            adr_src, base_src, adr_dest, base_dest,

   549                                            copy_type, value_type, count);

   550 

   551   Node* ctl = NULL;

   552   if (!forward_ctl->is_top() && !backward_ctl->is_top()) {

   553     ctl = new RegionNode(3);

   554     mem = new PhiNode(ctl, Type::MEMORY, atp_dest);

   555     ctl->init_req(1, forward_ctl);

   556     mem->init_req(1, forward_mem);

   557     ctl->init_req(2, backward_ctl);

   558     mem->init_req(2, backward_mem);

   559     ctl = phase->transform(ctl);

   560     mem = phase->transform(mem);

   561   } else if (!forward_ctl->is_top()) {

   562     ctl = forward_ctl;

   563     mem = forward_mem;

   564   } else {

   565     assert(!backward_ctl->is_top(), "no copy?");

   566     ctl = backward_ctl;

   567     mem = backward_mem;

   568   }

   569 

   570   if (can_reshape) {

   571     assert(phase->is_IterGVN()->delay_transform(), "should be delaying transforms");

   572     phase->is_IterGVN()->set_delay_transform(false);

   573   }

   574 

   575   MergeMemNode* out_mem = MergeMemNode::make(in_mem);

   576   out_mem->set_memory_at(alias_idx_dest, mem);

   577   mem = out_mem;

   578 

   579   if (!finish_transform(phase, can_reshape, ctl, mem)) {

   580     return NULL;

   581   }

   582 

   583   return mem;

   584 }