1 /*

     2  * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.

     3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

     4  *

     5  * This code is free software; you can redistribute it and/or modify it

     6  * under the terms of the GNU General Public License version 2 only, as

     7  * published by the Free Software Foundation.

     8  *

     9  * This code is distributed in the hope that it will be useful, but WITHOUT

    10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    12  * version 2 for more details (a copy is included in the LICENSE file that

    13  * accompanied this code).

    14  *

    15  * You should have received a copy of the GNU General Public License version

    16  * 2 along with this work; if not, write to the Free Software Foundation,

    17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    18  *

    19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

    20  * or visit www.oracle.com if you need additional information or have any

    21  * questions.

    22  */

    23 

    24 #include "precompiled.hpp"

    25 #include "opto/compile.hpp"

    26 #include "opto/castnode.hpp"

    27 #include "opto/graphKit.hpp"

    28 #include "opto/idealKit.hpp"

    29 #include "opto/loopnode.hpp"

    30 #include "opto/macro.hpp"

    31 #include "opto/node.hpp"

    32 #include "opto/type.hpp"

    33 #include "utilities/macros.hpp"

    34 #include "gc/z/c2/zBarrierSetC2.hpp"

    35 #include "gc/z/zThreadLocalData.hpp"

    36 #include "gc/z/zBarrierSetRuntime.hpp"

    37 

    38 ZBarrierSetC2State::ZBarrierSetC2State(Arena* comp_arena)

    39   : _load_barrier_nodes(new (comp_arena) GrowableArray<LoadBarrierNode*>(comp_arena, 8,  0, NULL)) {}

    40 

    41 int ZBarrierSetC2State::load_barrier_count() const {

    42   return _load_barrier_nodes->length();

    43 }

    44 

    45 void ZBarrierSetC2State::add_load_barrier_node(LoadBarrierNode * n) {

    46   assert(!_load_barrier_nodes->contains(n), " duplicate entry in expand list");

    47   _load_barrier_nodes->append(n);

    48 }

    49 

    50 void ZBarrierSetC2State::remove_load_barrier_node(LoadBarrierNode * n) {

    51   // this function may be called twice for a node so check

    52   // that the node is in the array before attempting to remove it

    53   if (_load_barrier_nodes->contains(n)) {

    54     _load_barrier_nodes->remove(n);

    55   }

    56 }

    57 

    58 LoadBarrierNode* ZBarrierSetC2State::load_barrier_node(int idx) const {

    59   return _load_barrier_nodes->at(idx);

    60 }

    61 

    62 void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const {

    63   return new(comp_arena) ZBarrierSetC2State(comp_arena);

    64 }

    65 

    66 ZBarrierSetC2State* ZBarrierSetC2::state() const {

    67   return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state());

    68 }

    69 

    70 bool ZBarrierSetC2::is_gc_barrier_node(Node* node) const {

    71   return node->is_LoadBarrier();

    72 }

    73 

    74 void ZBarrierSetC2::register_potential_barrier_node(Node* node) const {

    75   if (node->is_LoadBarrier()) {

    76     state()->add_load_barrier_node(node->as_LoadBarrier());

    77   }

    78 }

    79 

    80 void ZBarrierSetC2::unregister_potential_barrier_node(Node* node) const {

    81   if (node->is_LoadBarrier()) {

    82     state()->remove_load_barrier_node(node->as_LoadBarrier());

    83   }

    84 }

    85 

    86 void ZBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful) const {

    87   // Remove useless LoadBarrier nodes

    88   ZBarrierSetC2State* s = state();

    89   for (int i = s->load_barrier_count()-1; i >= 0; i--) {

    90     LoadBarrierNode* n = s->load_barrier_node(i);

    91     if (!useful.member(n)) {

    92       unregister_potential_barrier_node(n);

    93     }

    94   }

    95 }

    96 

    97 void ZBarrierSetC2::enqueue_useful_gc_barrier(Unique_Node_List &worklist, Node* node) const {

    98   if (node->is_LoadBarrier() && !node->as_LoadBarrier()->has_true_uses()) {

    99     worklist.push(node);

   100   }

   101 }

   102 

   103 void ZBarrierSetC2::find_dominating_barriers(PhaseIterGVN& igvn) {

   104   // Look for dominating barriers on the same address only once all

   105   // other loop opts are over: loop opts may cause a safepoint to be

   106   // inserted between a barrier and its dominating barrier.

   107   Compile* C = Compile::current();

   108   ZBarrierSetC2* bs = (ZBarrierSetC2*)BarrierSet::barrier_set()->barrier_set_c2();

   109   ZBarrierSetC2State* s = bs->state();

   110   if (s->load_barrier_count() >= 2) {

   111     Compile::TracePhase tp("idealLoop", &C->timers[Phase::_t_idealLoop]);

   112     PhaseIdealLoop ideal_loop(igvn, true, false, true);

   113     if (C->major_progress()) C->print_method(PHASE_PHASEIDEALLOOP_ITERATIONS, 2);

   114   }

   115 }

   116 

   117 void ZBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {

   118   // Permanent temporary workaround

   119   // Loadbarriers may have non-obvious dead uses keeping them alive during parsing. The use is

   120   // removed by RemoveUseless (after parsing, before optimize) but the barriers won't be added to

   121   // the worklist. Unless we add them explicitly they are not guaranteed to end up there.

   122   ZBarrierSetC2State* s = state();

   123 

   124   for (int i = 0; i < s->load_barrier_count(); i++) {

   125     LoadBarrierNode* n = s->load_barrier_node(i);

   126     worklist->push(n);

   127   }

   128 }

   129 

   130 const TypeFunc* ZBarrierSetC2::load_barrier_Type() const {

   131   const Type** fields;

   132 

   133   // Create input types (domain)

   134   fields = TypeTuple::fields(2);

   135   fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL;

   136   fields[TypeFunc::Parms+1] = TypeOopPtr::BOTTOM;

   137   const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);

   138 

   139   // Create result type (range)

   140   fields = TypeTuple::fields(1);

   141   fields[TypeFunc::Parms+0] = TypeInstPtr::BOTTOM;

   142   const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);

   143 

   144   return TypeFunc::make(domain, range);

   145 }

   146 

   147 // == LoadBarrierNode ==

   148 

   149 LoadBarrierNode::LoadBarrierNode(Compile* C,

   150                                  Node* c,

   151                                  Node* mem,

   152                                  Node* val,

   153                                  Node* adr,

   154                                  bool weak,

   155                                  bool writeback,

   156                                  bool oop_reload_allowed) :

   157     MultiNode(Number_of_Inputs),

   158     _weak(weak),

   159     _writeback(writeback),

   160     _oop_reload_allowed(oop_reload_allowed) {

   161   init_req(Control, c);

   162   init_req(Memory, mem);

   163   init_req(Oop, val);

   164   init_req(Address, adr);

   165   init_req(Similar, C->top());

   166 

   167   init_class_id(Class_LoadBarrier);

   168   BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();

   169   bs->register_potential_barrier_node(this);

   170 }

   171 

   172 const Type *LoadBarrierNode::bottom_type() const {

   173   const Type** floadbarrier = (const Type **)(Compile::current()->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*)));

   174   Node* in_oop = in(Oop);

   175   floadbarrier[Control] = Type::CONTROL;

   176   floadbarrier[Memory] = Type::MEMORY;

   177   floadbarrier[Oop] = in_oop == NULL ? Type::TOP : in_oop->bottom_type();

   178   return TypeTuple::make(Number_of_Outputs, floadbarrier);

   179 }

   180 

   181 const Type *LoadBarrierNode::Value(PhaseGVN *phase) const {

   182   const Type** floadbarrier = (const Type **)(phase->C->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*)));

   183   const Type* val_t = phase->type(in(Oop));

   184   floadbarrier[Control] = Type::CONTROL;

   185   floadbarrier[Memory] = Type::MEMORY;

   186   floadbarrier[Oop] = val_t;

   187   return TypeTuple::make(Number_of_Outputs, floadbarrier);

   188 }

   189 

   190 bool LoadBarrierNode::is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n) {

   191   if (phase != NULL) {

   192     return phase->is_dominator(d, n);

   193   }

   194 

   195   for (int i = 0; i < 10 && n != NULL; i++) {

   196     n = IfNode::up_one_dom(n, linear_only);

   197     if (n == d) {

   198       return true;

   199     }

   200   }

   201 

   202   return false;

   203 }

   204 

   205 LoadBarrierNode* LoadBarrierNode::has_dominating_barrier(PhaseIdealLoop* phase, bool linear_only, bool look_for_similar) {

   206   Node* val = in(LoadBarrierNode::Oop);

   207   if (in(Similar)->is_Proj() && in(Similar)->in(0)->is_LoadBarrier()) {

   208     LoadBarrierNode* lb = in(Similar)->in(0)->as_LoadBarrier();

   209     assert(lb->in(Address) == in(Address), "");

   210     // Load barrier on Similar edge dominates so if it now has the Oop field it can replace this barrier.

   211     if (lb->in(Oop) == in(Oop)) {

   212       return lb;

   213     }

   214     // Follow chain of load barrier through Similar edges

   215     while (!lb->in(Similar)->is_top()) {

   216       lb = lb->in(Similar)->in(0)->as_LoadBarrier();

   217       assert(lb->in(Address) == in(Address), "");

   218     }

   219     if (lb != in(Similar)->in(0)) {

   220       return lb;

   221     }

   222   }

   223   for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {

   224     Node* u = val->fast_out(i);

   225     if (u != this && u->is_LoadBarrier() && u->in(Oop) == val && u->as_LoadBarrier()->has_true_uses()) {

   226       Node* this_ctrl = in(LoadBarrierNode::Control);

   227       Node* other_ctrl = u->in(LoadBarrierNode::Control);

   228       if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) {

   229         return u->as_LoadBarrier();

   230       }

   231     }

   232   }

   233 

   234   if (ZVerifyLoadBarriers || can_be_eliminated()) {

   235     return NULL;

   236   }

   237 

   238   if (!look_for_similar) {

   239     return NULL;

   240   }

   241 

   242   Node* addr = in(LoadBarrierNode::Address);

   243   for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) {

   244     Node* u = addr->fast_out(i);

   245     if (u != this && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) {

   246       Node* this_ctrl = in(LoadBarrierNode::Control);

   247       Node* other_ctrl = u->in(LoadBarrierNode::Control);

   248       if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) {

   249         ResourceMark rm;

   250         Unique_Node_List wq;

   251         wq.push(in(LoadBarrierNode::Control));

   252         bool ok = true;

   253         bool dom_found = false;

   254         for (uint next = 0; next < wq.size(); ++next) {

   255           Node *n = wq.at(next);

   256           if (n->is_top()) {

   257             return NULL;

   258           }

   259           assert(n->is_CFG(), "");

   260           if (n->is_SafePoint()) {

   261             ok = false;

   262             break;

   263           }

   264           if (n == u) {

   265             dom_found = true;

   266             continue;

   267           }

   268           if (n->is_Region()) {

   269             for (uint i = 1; i < n->req(); i++) {

   270               Node* m = n->in(i);

   271               if (m != NULL) {

   272                 wq.push(m);

   273               }

   274             }

   275           } else {

   276             Node* m = n->in(0);

   277             if (m != NULL) {

   278               wq.push(m);

   279             }

   280           }

   281         }

   282         if (ok) {

   283           assert(dom_found, "");

   284           return u->as_LoadBarrier();;

   285         }

   286         break;

   287       }

   288     }

   289   }

   290 

   291   return NULL;

   292 }

   293 

   294 void LoadBarrierNode::push_dominated_barriers(PhaseIterGVN* igvn) const {

   295   // change to that barrier may affect a dominated barrier so re-push those

   296   Node* val = in(LoadBarrierNode::Oop);

   297 

   298   for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {

   299     Node* u = val->fast_out(i);

   300     if (u != this && u->is_LoadBarrier() && u->in(Oop) == val) {

   301       Node* this_ctrl = in(Control);

   302       Node* other_ctrl = u->in(Control);

   303       if (is_dominator(NULL, false, this_ctrl, other_ctrl)) {

   304         igvn->_worklist.push(u);

   305       }

   306     }

   307 

   308     Node* addr = in(LoadBarrierNode::Address);

   309     for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) {

   310       Node* u = addr->fast_out(i);

   311       if (u != this && u->is_LoadBarrier() && u->in(Similar)->is_top()) {

   312         Node* this_ctrl = in(Control);

   313         Node* other_ctrl = u->in(Control);

   314         if (is_dominator(NULL, false, this_ctrl, other_ctrl)) {

   315           igvn->_worklist.push(u);

   316         }

   317       }

   318     }

   319   }

   320 }

   321 

   322 Node *LoadBarrierNode::Identity(PhaseGVN *phase) {

   323   if (!phase->C->directive()->ZOptimizeLoadBarriersOption) {

   324     return this;

   325   }

   326 

   327   bool redundant_addr = false;

   328   LoadBarrierNode* dominating_barrier = has_dominating_barrier(NULL, true, false);

   329   if (dominating_barrier != NULL) {

   330     assert(dominating_barrier->in(Oop) == in(Oop), "");

   331     return dominating_barrier;

   332   }

   333 

   334   return this;

   335 }

   336 

   337 Node *LoadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) {

   338   if (remove_dead_region(phase, can_reshape)) {

   339     return this;

   340   }

   341 

   342   Node* val = in(Oop);

   343   Node* mem = in(Memory);

   344   Node* ctrl = in(Control);

   345   Node* adr = in(Address);

   346   assert(val->Opcode() != Op_LoadN, "");

   347 

   348   if (mem->is_MergeMem()) {

   349     Node* new_mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw);

   350     set_req(Memory, new_mem);

   351     if (mem->outcnt() == 0 && can_reshape) {

   352       phase->is_IterGVN()->_worklist.push(mem);

   353     }

   354 

   355     return this;

   356   }

   357 

   358   bool optimizeLoadBarriers = phase->C->directive()->ZOptimizeLoadBarriersOption;

   359   LoadBarrierNode* dominating_barrier = optimizeLoadBarriers ? has_dominating_barrier(NULL, !can_reshape, !phase->C->major_progress()) : NULL;

   360   if (dominating_barrier != NULL && dominating_barrier->in(Oop) != in(Oop)) {

   361     assert(in(Address) == dominating_barrier->in(Address), "");

   362     set_req(Similar, dominating_barrier->proj_out(Oop));

   363     return this;

   364   }

   365 

   366   bool eliminate = (optimizeLoadBarriers && !(val->is_Phi() || val->Opcode() == Op_LoadP || val->Opcode() == Op_GetAndSetP || val->is_DecodeN())) ||

   367                    (can_reshape && (dominating_barrier != NULL || !has_true_uses()));

   368 

   369   if (eliminate) {

   370     if (can_reshape) {

   371       PhaseIterGVN* igvn = phase->is_IterGVN();

   372       Node* out_ctrl = proj_out_or_null(Control);

   373       Node* out_res = proj_out_or_null(Oop);

   374 

   375       if (out_ctrl != NULL) {

   376         igvn->replace_node(out_ctrl, ctrl);

   377       }

   378 

   379       // That transformation may cause the Similar edge on the load barrier to be invalid

   380       fix_similar_in_uses(igvn);

   381       if (out_res != NULL) {

   382         if (dominating_barrier != NULL) {

   383           igvn->replace_node(out_res, dominating_barrier->proj_out(Oop));

   384         } else {

   385           igvn->replace_node(out_res, val);

   386         }

   387       }

   388     }

   389 

   390     return new ConINode(TypeInt::ZERO);

   391   }

   392 

   393   // If the Similar edge is no longer a load barrier, clear it

   394   Node* similar = in(Similar);

   395   if (!similar->is_top() && !(similar->is_Proj() && similar->in(0)->is_LoadBarrier())) {

   396     set_req(Similar, phase->C->top());

   397     return this;

   398   }

   399 

   400   if (can_reshape) {

   401     // If this barrier is linked through the Similar edge by a

   402     // dominated barrier and both barriers have the same Oop field,

   403     // the dominated barrier can go away, so push it for reprocessing.

   404     // We also want to avoid a barrier to depend on another dominating

   405     // barrier through its Similar edge that itself depend on another

   406     // barrier through its Similar edge and rather have the first

   407     // depend on the third.

   408     PhaseIterGVN* igvn = phase->is_IterGVN();

   409     Node* out_res = proj_out(Oop);

   410     for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {

   411       Node* u = out_res->fast_out(i);

   412       if (u->is_LoadBarrier() && u->in(Similar) == out_res &&

   413           (u->in(Oop) == val || !u->in(Similar)->is_top())) {

   414         igvn->_worklist.push(u);

   415       }

   416     }

   417 

   418     push_dominated_barriers(igvn);

   419   }

   420 

   421   return NULL;

   422 }

   423 

   424 void LoadBarrierNode::fix_similar_in_uses(PhaseIterGVN* igvn) {

   425   Node* out_res = proj_out_or_null(Oop);

   426   if (out_res == NULL) {

   427     return;

   428   }

   429 

   430   for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {

   431     Node* u = out_res->fast_out(i);

   432     if (u->is_LoadBarrier() && u->in(Similar) == out_res) {

   433       igvn->replace_input_of(u, Similar, igvn->C->top());

   434       --i;

   435       --imax;

   436     }

   437   }

   438 }

   439 

   440 bool LoadBarrierNode::has_true_uses() const {

   441   Node* out_res = proj_out_or_null(Oop);

   442   if (out_res == NULL) {

   443     return false;

   444   }

   445 

   446   for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {

   447     Node* u = out_res->fast_out(i);

   448     if (!u->is_LoadBarrier() || u->in(Similar) != out_res) {

   449       return true;

   450     }

   451   }

   452 

   453   return false;

   454 }

   455 

   456 // == Accesses ==

   457 

   458 Node* ZBarrierSetC2::make_cas_loadbarrier(C2AtomicAccess& access) const {

   459   assert(!UseCompressedOops, "Not allowed");

   460   CompareAndSwapNode* cas = (CompareAndSwapNode*)access.raw_access();

   461   PhaseGVN& gvn = access.kit()->gvn();

   462   Compile* C = Compile::current();

   463   GraphKit* kit = access.kit();

   464 

   465   Node* in_ctrl     = cas->in(MemNode::Control);

   466   Node* in_mem      = cas->in(MemNode::Memory);

   467   Node* in_adr      = cas->in(MemNode::Address);

   468   Node* in_val      = cas->in(MemNode::ValueIn);

   469   Node* in_expected = cas->in(LoadStoreConditionalNode::ExpectedIn);

   470 

   471   float likely                   = PROB_LIKELY(0.999);

   472 

   473   const TypePtr *adr_type        = gvn.type(in_adr)->isa_ptr();

   474   Compile::AliasType* alias_type = C->alias_type(adr_type);

   475   int alias_idx                  = C->get_alias_index(adr_type);

   476 

   477   // Outer check - true: continue, false: load and check

   478   Node* region   = new RegionNode(3);

   479   Node* phi      = new PhiNode(region, TypeInt::BOOL);

   480   Node* phi_mem  = new PhiNode(region, Type::MEMORY, adr_type);

   481 

   482   // Inner check - is the healed ref equal to the expected

   483   Node* region2  = new RegionNode(3);

   484   Node* phi2     = new PhiNode(region2, TypeInt::BOOL);

   485   Node* phi_mem2 = new PhiNode(region2, Type::MEMORY, adr_type);

   486 

   487   // CAS node returns 0 or 1

   488   Node* cmp     = gvn.transform(new CmpINode(cas, kit->intcon(0)));

   489   Node* bol     = gvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();

   490   IfNode* iff   = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If();

   491   Node* then    = gvn.transform(new IfTrueNode(iff));

   492   Node* elsen   = gvn.transform(new IfFalseNode(iff));

   493 

   494   Node* scmemproj1   = gvn.transform(new SCMemProjNode(cas));

   495 

   496   kit->set_memory(scmemproj1, alias_idx);

   497   phi_mem->init_req(1, scmemproj1);

   498   phi_mem2->init_req(2, scmemproj1);

   499 

   500   // CAS fail - reload and heal oop

   501   Node* reload      = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered);

   502   Node* barrier     = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false));

   503   Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control));

   504   Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop));

   505 

   506   // Check load

   507   Node* tmpX    = gvn.transform(new CastP2XNode(NULL, barrierdata));

   508   Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected));

   509   Node* cmp2    = gvn.transform(new CmpXNode(tmpX, in_expX));

   510   Node *bol2    = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool();

   511   IfNode* iff2  = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If();

   512   Node* then2   = gvn.transform(new IfTrueNode(iff2));

   513   Node* elsen2  = gvn.transform(new IfFalseNode(iff2));

   514 

   515   // redo CAS

   516   Node* cas2       = gvn.transform(new CompareAndSwapPNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, cas->order()));

   517   Node* scmemproj2 = gvn.transform(new SCMemProjNode(cas2));

   518   kit->set_control(elsen2);

   519   kit->set_memory(scmemproj2, alias_idx);

   520 

   521   // Merge inner flow - check if healed oop was equal too expected.

   522   region2->set_req(1, kit->control());

   523   region2->set_req(2, then2);

   524   phi2->set_req(1, cas2);

   525   phi2->set_req(2, kit->intcon(0));

   526   phi_mem2->init_req(1, scmemproj2);

   527   kit->set_memory(phi_mem2, alias_idx);

   528 

   529   // Merge outer flow - then check if first cas succeded

   530   region->set_req(1, then);

   531   region->set_req(2, region2);

   532   phi->set_req(1, kit->intcon(1));

   533   phi->set_req(2, phi2);

   534   phi_mem->init_req(2, phi_mem2);

   535   kit->set_memory(phi_mem, alias_idx);

   536 

   537   gvn.transform(region2);

   538   gvn.transform(phi2);

   539   gvn.transform(phi_mem2);

   540   gvn.transform(region);

   541   gvn.transform(phi);

   542   gvn.transform(phi_mem);

   543 

   544   kit->set_control(region);

   545   kit->insert_mem_bar(Op_MemBarCPUOrder);

   546 

   547   return phi;

   548 }

   549 

   550 Node* ZBarrierSetC2::make_cmpx_loadbarrier(C2AtomicAccess& access) const {

   551   CompareAndExchangePNode* cmpx = (CompareAndExchangePNode*)access.raw_access();

   552   GraphKit* kit = access.kit();

   553   PhaseGVN& gvn = kit->gvn();

   554   Compile* C = Compile::current();

   555 

   556   Node* in_ctrl     = cmpx->in(MemNode::Control);

   557   Node* in_mem      = cmpx->in(MemNode::Memory);

   558   Node* in_adr      = cmpx->in(MemNode::Address);

   559   Node* in_val      = cmpx->in(MemNode::ValueIn);

   560   Node* in_expected = cmpx->in(LoadStoreConditionalNode::ExpectedIn);

   561 

   562   float likely                   = PROB_LIKELY(0.999);

   563 

   564   const TypePtr *adr_type        = cmpx->get_ptr_type();

   565   Compile::AliasType* alias_type = C->alias_type(adr_type);

   566   int alias_idx                  = C->get_alias_index(adr_type);

   567 

   568   // Outer check - true: continue, false: load and check

   569   Node* region  = new RegionNode(3);

   570   Node* phi     = new PhiNode(region, adr_type);

   571 

   572   // Inner check - is the healed ref equal to the expected

   573   Node* region2 = new RegionNode(3);

   574   Node* phi2    = new PhiNode(region2, adr_type);

   575 

   576   // Check if cmpx succeded

   577   Node* cmp     = gvn.transform(new CmpPNode(cmpx, in_expected));

   578   Node* bol     = gvn.transform(new BoolNode(cmp, BoolTest::eq))->as_Bool();

   579   IfNode* iff   = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If();

   580   Node* then    = gvn.transform(new IfTrueNode(iff));

   581   Node* elsen   = gvn.transform(new IfFalseNode(iff));

   582 

   583   Node* scmemproj1  = gvn.transform(new SCMemProjNode(cmpx));

   584   kit->set_memory(scmemproj1, alias_idx);

   585 

   586   // CAS fail - reload and heal oop

   587   Node* reload      = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered);

   588   Node* barrier     = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false));

   589   Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control));

   590   Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop));

   591 

   592   // Check load

   593   Node* tmpX    = gvn.transform(new CastP2XNode(NULL, barrierdata));

   594   Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected));

   595   Node* cmp2    = gvn.transform(new CmpXNode(tmpX, in_expX));

   596   Node *bol2    = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool();

   597   IfNode* iff2  = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If();

   598   Node* then2   = gvn.transform(new IfTrueNode(iff2));

   599   Node* elsen2  = gvn.transform(new IfFalseNode(iff2));

   600 

   601   // Redo CAS

   602   Node* cmpx2      = gvn.transform(new CompareAndExchangePNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, adr_type, cmpx->get_ptr_type(), cmpx->order()));

   603   Node* scmemproj2 = gvn.transform(new SCMemProjNode(cmpx2));

   604   kit->set_control(elsen2);

   605   kit->set_memory(scmemproj2, alias_idx);

   606 

   607   // Merge inner flow - check if healed oop was equal too expected.

   608   region2->set_req(1, kit->control());

   609   region2->set_req(2, then2);

   610   phi2->set_req(1, cmpx2);

   611   phi2->set_req(2, barrierdata);

   612 

   613   // Merge outer flow - then check if first cas succeded

   614   region->set_req(1, then);

   615   region->set_req(2, region2);

   616   phi->set_req(1, cmpx);

   617   phi->set_req(2, phi2);

   618 

   619   gvn.transform(region2);

   620   gvn.transform(phi2);

   621   gvn.transform(region);

   622   gvn.transform(phi);

   623 

   624   kit->set_control(region);

   625   kit->set_memory(in_mem, alias_idx);

   626   kit->insert_mem_bar(Op_MemBarCPUOrder);

   627 

   628   return phi;

   629 }

   630 

   631 Node* ZBarrierSetC2::load_barrier(GraphKit* kit, Node* val, Node* adr, bool weak, bool writeback, bool oop_reload_allowed) const {

   632   PhaseGVN& gvn = kit->gvn();

   633   Node* barrier = new LoadBarrierNode(Compile::current(), kit->control(), kit->memory(TypeRawPtr::BOTTOM), val, adr, weak, writeback, oop_reload_allowed);

   634   Node* transformed_barrier = gvn.transform(barrier);

   635 

   636   if (transformed_barrier->is_LoadBarrier()) {

   637     if (barrier == transformed_barrier) {

   638       kit->set_control(gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control)));

   639     }

   640     return gvn.transform(new ProjNode(transformed_barrier, LoadBarrierNode::Oop));

   641   } else {

   642     return val;

   643   }

   644 }

   645 

   646 static bool barrier_needed(C2Access access) {

   647   return ZBarrierSet::barrier_needed(access.decorators(), access.type());

   648 }

   649 

   650 Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {

   651   Node* p = BarrierSetC2::load_at_resolved(access, val_type);

   652   if (!barrier_needed(access)) {

   653     return p;

   654   }

   655 

   656   bool conc_root = (access.decorators() & IN_CONCURRENT_ROOT) != 0;

   657   bool weak = (access.decorators() & ON_WEAK_OOP_REF) != 0;

   658 

   659   GraphKit* kit = access.kit();

   660   PhaseGVN& gvn = kit->gvn();

   661   Node* adr = access.addr().node();

   662   Node* heap_base_oop = access.base();

   663   bool unsafe = (access.decorators() & C2_UNSAFE_ACCESS) != 0;

   664   if (unsafe) {

   665     if (!ZVerifyLoadBarriers) {

   666       p = load_barrier(kit, p, adr);

   667     } else {

   668       if (!TypePtr::NULL_PTR->higher_equal(gvn.type(heap_base_oop))) {

   669         p = load_barrier(kit, p, adr);

   670       } else {

   671         IdealKit ideal(kit);

   672         IdealVariable res(ideal);

   673 #define __ ideal.

   674         __ declarations_done();

   675         __ set(res, p);

   676         __ if_then(heap_base_oop, BoolTest::ne, kit->null(), PROB_UNLIKELY(0.999)); {

   677           kit->sync_kit(ideal);

   678           p = load_barrier(kit, p, adr);

   679           __ set(res, p);

   680           __ sync_kit(kit);

   681         } __ end_if();

   682         kit->final_sync(ideal);

   683         p = __ value(res);

   684 #undef __

   685       }

   686     }

   687     return p;

   688   } else {

   689     return load_barrier(access.kit(), p, access.addr().node(), weak, true, true);

   690   }

   691 }

   692 

   693 Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val,

   694                                                     Node* new_val, const Type* val_type) const {

   695   Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type);

   696   if (!barrier_needed(access)) {

   697     return result;

   698   }

   699 

   700   access.set_needs_pinning(false);

   701   return make_cmpx_loadbarrier(access);

   702 }

   703 

   704 Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val,

   705                                                      Node* new_val, const Type* value_type) const {

   706   Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);

   707   if (!barrier_needed(access)) {

   708     return result;

   709   }

   710 

   711   Node* load_store = access.raw_access();

   712   bool weak_cas = (access.decorators() & C2_WEAK_CMPXCHG) != 0;

   713   bool expected_is_null = (expected_val->get_ptr_type() == TypePtr::NULL_PTR);

   714 

   715   if (!expected_is_null) {

   716     if (weak_cas) {

   717       access.set_needs_pinning(false);

   718       load_store = make_cas_loadbarrier(access);

   719     } else {

   720       access.set_needs_pinning(false);

   721       load_store = make_cas_loadbarrier(access);

   722     }

   723   }

   724 

   725   return load_store;

   726 }

   727 

   728 Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* val_type) const {

   729   Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type);

   730   if (!barrier_needed(access)) {

   731     return result;

   732   }

   733 

   734   Node* load_store = access.raw_access();

   735   Node* adr = access.addr().node();

   736 

   737   return load_barrier(access.kit(), load_store, adr, false, false, false);

   738 }

   739 

   740 // == Macro Expansion ==

   741 

   742 void ZBarrierSetC2::expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const {

   743   Node* in_ctrl = barrier->in(LoadBarrierNode::Control);

   744   Node* in_mem  = barrier->in(LoadBarrierNode::Memory);

   745   Node* in_val  = barrier->in(LoadBarrierNode::Oop);

   746   Node* in_adr  = barrier->in(LoadBarrierNode::Address);

   747 

   748   Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);

   749   Node* out_res  = barrier->proj_out(LoadBarrierNode::Oop);

   750 

   751   PhaseIterGVN &igvn = phase->igvn();

   752 

   753   if (ZVerifyLoadBarriers) {

   754     igvn.replace_node(out_res, in_val);

   755     igvn.replace_node(out_ctrl, in_ctrl);

   756     return;

   757   }

   758 

   759   if (barrier->can_be_eliminated()) {

   760     // Clone and pin the load for this barrier below the dominating

   761     // barrier: the load cannot be allowed to float above the

   762     // dominating barrier

   763     Node* load = in_val;

   764 

   765     if (load->is_Load()) {

   766       Node* new_load = load->clone();

   767       Node* addp = new_load->in(MemNode::Address);

   768       assert(addp->is_AddP() || addp->is_Phi() || addp->is_Load(), "bad address");

   769       Node* cast = new CastPPNode(addp, igvn.type(addp), true);

   770       Node* ctrl = NULL;

   771       Node* similar = barrier->in(LoadBarrierNode::Similar);

   772       if (similar->is_Phi()) {

   773         // already expanded

   774         ctrl = similar->in(0);

   775       } else {

   776         assert(similar->is_Proj() && similar->in(0)->is_LoadBarrier(), "unexpected graph shape");

   777         ctrl = similar->in(0)->as_LoadBarrier()->proj_out(LoadBarrierNode::Control);

   778       }

   779       assert(ctrl != NULL, "bad control");

   780       cast->set_req(0, ctrl);

   781       igvn.transform(cast);

   782       new_load->set_req(MemNode::Address, cast);

   783       igvn.transform(new_load);

   784 

   785       igvn.replace_node(out_res, new_load);

   786       igvn.replace_node(out_ctrl, in_ctrl);

   787       return;

   788     }

   789     // cannot eliminate

   790   }

   791 

   792   // There are two cases that require the basic loadbarrier

   793   // 1) When the writeback of a healed oop must be avoided (swap)

   794   // 2) When we must guarantee that no reload of is done (swap, cas, cmpx)

   795   if (!barrier->is_writeback()) {

   796     assert(!barrier->oop_reload_allowed(), "writeback barriers should be marked as requires oop");

   797   }

   798 

   799   if (!barrier->oop_reload_allowed()) {

   800     expand_loadbarrier_basic(phase, barrier);

   801   } else {

   802     expand_loadbarrier_optimized(phase, barrier);

   803   }

   804 }

   805 

   806 // Basic loadbarrier using conventional arg passing

   807 void ZBarrierSetC2::expand_loadbarrier_basic(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const {

   808   PhaseIterGVN &igvn = phase->igvn();

   809 

   810   Node* in_ctrl = barrier->in(LoadBarrierNode::Control);

   811   Node* in_mem  = barrier->in(LoadBarrierNode::Memory);

   812   Node* in_val  = barrier->in(LoadBarrierNode::Oop);

   813   Node* in_adr  = barrier->in(LoadBarrierNode::Address);

   814 

   815   Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);

   816   Node* out_res  = barrier->proj_out(LoadBarrierNode::Oop);

   817 

   818   float unlikely  = PROB_UNLIKELY(0.999);

   819   const Type* in_val_maybe_null_t = igvn.type(in_val);

   820 

   821   Node* jthread = igvn.transform(new ThreadLocalNode());

   822   Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset()));

   823   Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr, TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), MemNode::unordered));

   824   Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val));

   825   Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask));

   826   Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type())));

   827   Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();

   828   IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If();

   829   Node* then = igvn.transform(new IfTrueNode(iff));

   830   Node* elsen = igvn.transform(new IfFalseNode(iff));

   831 

   832   Node* result_region;

   833   Node* result_val;

   834 

   835   result_region = new RegionNode(3);

   836   result_val = new PhiNode(result_region, TypeInstPtr::BOTTOM);

   837 

   838   result_region->set_req(1, elsen);

   839   Node* res = igvn.transform(new CastPPNode(in_val, in_val_maybe_null_t));

   840   res->init_req(0, elsen);

   841   result_val->set_req(1, res);

   842 

   843   const TypeFunc *tf = load_barrier_Type();

   844   Node* call;

   845   if (barrier->is_weak()) {

   846     call = new CallLeafNode(tf,

   847                             ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr(),

   848                             "ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded",

   849                             TypeRawPtr::BOTTOM);

   850   } else {

   851     call = new CallLeafNode(tf,

   852                             ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(),

   853                             "ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded",

   854                             TypeRawPtr::BOTTOM);

   855   }

   856 

   857   call->init_req(TypeFunc::Control, then);

   858   call->init_req(TypeFunc::I_O    , phase->top());

   859   call->init_req(TypeFunc::Memory , in_mem);

   860   call->init_req(TypeFunc::FramePtr, phase->top());

   861   call->init_req(TypeFunc::ReturnAdr, phase->top());

   862   call->init_req(TypeFunc::Parms+0, in_val);

   863   if (barrier->is_writeback()) {

   864     call->init_req(TypeFunc::Parms+1, in_adr);

   865   } else {

   866     // when slow path is called with a null adr, the healed oop will not be written back

   867     call->init_req(TypeFunc::Parms+1, igvn.zerocon(T_OBJECT));

   868   }

   869   call = igvn.transform(call);

   870 

   871   Node* ctrl = igvn.transform(new ProjNode(call, TypeFunc::Control));

   872   res = igvn.transform(new ProjNode(call, TypeFunc::Parms));

   873   res = igvn.transform(new CheckCastPPNode(ctrl, res, in_val_maybe_null_t));

   874 

   875   result_region->set_req(2, ctrl);

   876   result_val->set_req(2, res);

   877 

   878   result_region = igvn.transform(result_region);

   879   result_val = igvn.transform(result_val);

   880 

   881   if (out_ctrl != NULL) { // added if cond

   882     igvn.replace_node(out_ctrl, result_region);

   883   }

   884   igvn.replace_node(out_res, result_val);

   885 }

   886 

   887 // Optimized, low spill, loadbarrier variant using stub specialized on register used

   888 void ZBarrierSetC2::expand_loadbarrier_optimized(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const {

   889   PhaseIterGVN &igvn = phase->igvn();

   890 #ifdef PRINT_NODE_TRAVERSALS

   891   Node* preceding_barrier_node = barrier->in(LoadBarrierNode::Oop);

   892 #endif

   893 

   894   Node* in_ctrl = barrier->in(LoadBarrierNode::Control);

   895   Node* in_mem = barrier->in(LoadBarrierNode::Memory);

   896   Node* in_val = barrier->in(LoadBarrierNode::Oop);

   897   Node* in_adr = barrier->in(LoadBarrierNode::Address);

   898 

   899   Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);

   900   Node* out_res = barrier->proj_out(LoadBarrierNode::Oop);

   901 

   902   assert(barrier->in(LoadBarrierNode::Oop) != NULL, "oop to loadbarrier node cannot be null");

   903 

   904 #ifdef PRINT_NODE_TRAVERSALS

   905   tty->print("\n\n\nBefore barrier optimization:\n");

   906   traverse(barrier, out_ctrl, out_res, -1);

   907 

   908   tty->print("\nBefore barrier optimization:  preceding_barrier_node\n");

   909   traverse(preceding_barrier_node, out_ctrl, out_res, -1);

   910 #endif

   911 

   912   float unlikely  = PROB_UNLIKELY(0.999);

   913 

   914   Node* jthread = igvn.transform(new ThreadLocalNode());

   915   Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset()));

   916   Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr,

   917                                                  TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(),

   918                                                  MemNode::unordered));

   919   Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val));

   920   Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask));

   921   Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type())));

   922   Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();

   923   IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If();

   924   Node* then = igvn.transform(new IfTrueNode(iff));

   925   Node* elsen = igvn.transform(new IfFalseNode(iff));

   926 

   927   Node* slow_path_surrogate;

   928   if (!barrier->is_weak()) {

   929     slow_path_surrogate = igvn.transform(new LoadBarrierSlowRegNode(then, in_mem, in_adr, in_val->adr_type(),

   930                                                                     (const TypePtr*) in_val->bottom_type(), MemNode::unordered));

   931   } else {

   932     slow_path_surrogate = igvn.transform(new LoadBarrierWeakSlowRegNode(then, in_mem, in_adr, in_val->adr_type(),

   933                                                                         (const TypePtr*) in_val->bottom_type(), MemNode::unordered));

   934   }

   935 

   936   Node *new_loadp;

   937   new_loadp = slow_path_surrogate;

   938   // create the final region/phi pair to converge cntl/data paths to downstream code

   939   Node* result_region = igvn.transform(new RegionNode(3));

   940   result_region->set_req(1, then);

   941   result_region->set_req(2, elsen);

   942 

   943   Node* result_phi = igvn.transform(new PhiNode(result_region, TypeInstPtr::BOTTOM));

   944   result_phi->set_req(1, new_loadp);

   945   result_phi->set_req(2, barrier->in(LoadBarrierNode::Oop));

   946 

   947   // finally, connect the original outputs to the barrier region and phi to complete the expansion/substitution

   948   // igvn.replace_node(out_ctrl, result_region);

   949   if (out_ctrl != NULL) { // added if cond

   950     igvn.replace_node(out_ctrl, result_region);

   951   }

   952   igvn.replace_node(out_res, result_phi);

   953 

   954   assert(barrier->outcnt() == 0,"LoadBarrier macro node has non-null outputs after expansion!");

   955 

   956 #ifdef PRINT_NODE_TRAVERSALS

   957   tty->print("\nAfter barrier optimization:  old out_ctrl\n");

   958   traverse(out_ctrl, out_ctrl, out_res, -1);

   959   tty->print("\nAfter barrier optimization:  old out_res\n");

   960   traverse(out_res, out_ctrl, out_res, -1);

   961   tty->print("\nAfter barrier optimization:  old barrier\n");

   962   traverse(barrier, out_ctrl, out_res, -1);

   963   tty->print("\nAfter barrier optimization:  preceding_barrier_node\n");

   964   traverse(preceding_barrier_node, result_region, result_phi, -1);

   965 #endif

   966 

   967   return;

   968 }

   969 

   970 bool ZBarrierSetC2::expand_macro_nodes(PhaseMacroExpand* macro) const {

   971   Compile* C = Compile::current();

   972   PhaseIterGVN &igvn = macro->igvn();

   973   ZBarrierSetC2State* s = state();

   974   if (s->load_barrier_count() > 0) {

   975 #ifdef ASSERT

   976     verify_gc_barriers(false);

   977 #endif

   978     igvn.set_delay_transform(true);

   979     int skipped = 0;

   980     while (s->load_barrier_count() > skipped) {

   981       int load_barrier_count = s->load_barrier_count();

   982       LoadBarrierNode * n = s->load_barrier_node(load_barrier_count-1-skipped);

   983       if (igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) {

   984         // node is unreachable, so don't try to expand it

   985         s->remove_load_barrier_node(n);

   986         continue;

   987       }

   988       if (!n->can_be_eliminated()) {

   989         skipped++;

   990         continue;

   991       }

   992       expand_loadbarrier_node(macro, n);

   993       assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list");

   994       if (C->failing())  return true;

   995     }

   996     while (s->load_barrier_count() > 0) {

   997       int load_barrier_count = s->load_barrier_count();

   998       LoadBarrierNode* n = s->load_barrier_node(load_barrier_count - 1);

   999       assert(!(igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())), "should have been processed already");

  1000       assert(!n->can_be_eliminated(), "should have been processed already");

  1001       expand_loadbarrier_node(macro, n);

  1002       assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list");

  1003       if (C->failing())  return true;

  1004     }

  1005     igvn.set_delay_transform(false);

  1006     igvn.optimize();

  1007     if (C->failing())  return true;

  1008   }

  1009   return false;

  1010 }

  1011 

  1012 // == Loop optimization ==

  1013 

  1014 static bool replace_with_dominating_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, bool last_round) {

  1015   PhaseIterGVN &igvn = phase->igvn();

  1016   Compile* C = Compile::current();

  1017 

  1018   LoadBarrierNode* lb2 = lb->has_dominating_barrier(phase, false, last_round);

  1019   if (lb2 != NULL) {

  1020     if (lb->in(LoadBarrierNode::Oop) != lb2->in(LoadBarrierNode::Oop)) {

  1021       assert(lb->in(LoadBarrierNode::Address) == lb2->in(LoadBarrierNode::Address), "");

  1022       igvn.replace_input_of(lb, LoadBarrierNode::Similar, lb2->proj_out(LoadBarrierNode::Oop));

  1023       C->set_major_progress();

  1024     } else  {

  1025       // That transformation may cause the Similar edge on dominated load barriers to be invalid

  1026       lb->fix_similar_in_uses(&igvn);

  1027 

  1028       Node* val = lb->proj_out(LoadBarrierNode::Oop);

  1029       assert(lb2->has_true_uses(), "");

  1030       assert(lb2->in(LoadBarrierNode::Oop) == lb->in(LoadBarrierNode::Oop), "");

  1031 

  1032       phase->lazy_update(lb, lb->in(LoadBarrierNode::Control));

  1033       phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control));

  1034       igvn.replace_node(val, lb2->proj_out(LoadBarrierNode::Oop));

  1035 

  1036       return true;

  1037     }

  1038   }

  1039   return false;

  1040 }

  1041 

  1042 static Node* find_dominating_memory(PhaseIdealLoop* phase, Node* mem, Node* dom, int i) {

  1043   assert(dom->is_Region() || i == -1, "");

  1044   Node* m = mem;

  1045   while(phase->is_dominator(dom, phase->has_ctrl(m) ? phase->get_ctrl(m) : m->in(0))) {

  1046     if (m->is_Mem()) {

  1047       assert(m->as_Mem()->adr_type() == TypeRawPtr::BOTTOM, "");

  1048       m = m->in(MemNode::Memory);

  1049     } else if (m->is_MergeMem()) {

  1050       m = m->as_MergeMem()->memory_at(Compile::AliasIdxRaw);

  1051     } else if (m->is_Phi()) {

  1052       if (m->in(0) == dom && i != -1) {

  1053         m = m->in(i);

  1054         break;

  1055       } else {

  1056         m = m->in(LoopNode::EntryControl);

  1057       }

  1058     } else if (m->is_Proj()) {

  1059       m = m->in(0);

  1060     } else if (m->is_SafePoint() || m->is_MemBar()) {

  1061       m = m->in(TypeFunc::Memory);

  1062     } else {

  1063 #ifdef ASSERT

  1064       m->dump();

  1065 #endif

  1066       ShouldNotReachHere();

  1067     }

  1068   }

  1069   return m;

  1070 }

  1071 

  1072 static LoadBarrierNode* clone_load_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* ctl, Node* mem, Node* oop_in) {

  1073   PhaseIterGVN &igvn = phase->igvn();

  1074   Compile* C = Compile::current();

  1075   Node* the_clone = lb->clone();

  1076   the_clone->set_req(LoadBarrierNode::Control, ctl);

  1077   the_clone->set_req(LoadBarrierNode::Memory, mem);

  1078   if (oop_in != NULL) {

  1079     the_clone->set_req(LoadBarrierNode::Oop, oop_in);

  1080   }

  1081 

  1082   LoadBarrierNode* new_lb = the_clone->as_LoadBarrier();

  1083   igvn.register_new_node_with_optimizer(new_lb);

  1084   IdealLoopTree *loop = phase->get_loop(new_lb->in(0));

  1085   phase->set_ctrl(new_lb, new_lb->in(0));

  1086   phase->set_loop(new_lb, loop);

  1087   phase->set_idom(new_lb, new_lb->in(0), phase->dom_depth(new_lb->in(0))+1);

  1088   if (!loop->_child) {

  1089     loop->_body.push(new_lb);

  1090   }

  1091 

  1092   Node* proj_ctl = new ProjNode(new_lb, LoadBarrierNode::Control);

  1093   igvn.register_new_node_with_optimizer(proj_ctl);

  1094   phase->set_ctrl(proj_ctl, proj_ctl->in(0));

  1095   phase->set_loop(proj_ctl, loop);

  1096   phase->set_idom(proj_ctl, new_lb, phase->dom_depth(new_lb)+1);

  1097   if (!loop->_child) {

  1098     loop->_body.push(proj_ctl);

  1099   }

  1100 

  1101   Node* proj_oop = new ProjNode(new_lb, LoadBarrierNode::Oop);

  1102   phase->register_new_node(proj_oop, new_lb);

  1103 

  1104   if (!new_lb->in(LoadBarrierNode::Similar)->is_top()) {

  1105     LoadBarrierNode* similar = new_lb->in(LoadBarrierNode::Similar)->in(0)->as_LoadBarrier();

  1106     if (!phase->is_dominator(similar, ctl)) {

  1107       igvn.replace_input_of(new_lb, LoadBarrierNode::Similar, C->top());

  1108     }

  1109   }

  1110 

  1111   return new_lb;

  1112 }

  1113 

  1114 static void replace_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* new_val) {

  1115   PhaseIterGVN &igvn = phase->igvn();

  1116   Node* val = lb->proj_out(LoadBarrierNode::Oop);

  1117   igvn.replace_node(val, new_val);

  1118   phase->lazy_update(lb, lb->in(LoadBarrierNode::Control));

  1119   phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control));

  1120 }

  1121 

  1122 static bool split_barrier_thru_phi(PhaseIdealLoop* phase, LoadBarrierNode* lb) {

  1123   PhaseIterGVN &igvn = phase->igvn();

  1124   Compile* C = Compile::current();

  1125 

  1126   if (lb->in(LoadBarrierNode::Oop)->is_Phi()) {

  1127     Node* oop_phi = lb->in(LoadBarrierNode::Oop);

  1128 

  1129     if (oop_phi->in(2) == oop_phi) {

  1130       // Ignore phis with only one input

  1131       return false;

  1132     }

  1133 

  1134     if (phase->is_dominator(phase->get_ctrl(lb->in(LoadBarrierNode::Address)),

  1135                             oop_phi->in(0)) && phase->get_ctrl(lb->in(LoadBarrierNode::Address)) != oop_phi->in(0)) {