25 #include "opto/castnode.hpp"

    25 #include "classfile/javaClasses.hpp"

    26 #include "gc/z/c2/zBarrierSetC2.hpp"

    27 #include "gc/z/zBarrierSet.hpp"

    28 #include "gc/z/zBarrierSetAssembler.hpp"

    29 #include "gc/z/zBarrierSetRuntime.hpp"

    30 #include "opto/block.hpp"

    35 #include "opto/phase.hpp"

    36 #include "opto/regalloc.hpp"

    36 #include "opto/phaseX.hpp"

    42 #include "gc/z/zBarrierSet.hpp"

    40 

    43 #include "gc/z/c2/zBarrierSetC2.hpp"

    41 class ZBarrierSetC2State : public ResourceObj {

    44 #include "gc/z/zThreadLocalData.hpp"

    42 private:

    45 #include "gc/z/zBarrierSetRuntime.hpp"

    43   GrowableArray<ZLoadBarrierStubC2*>* _stubs;

    46 

    44   Node_Array                          _live;

    47 ZBarrierSetC2State::ZBarrierSetC2State(Arena* comp_arena) :

    45 

    48     _load_barrier_nodes(new (comp_arena) GrowableArray<LoadBarrierNode*>(comp_arena, 8,  0, NULL)) {}

    46 public:

    49 

    47   ZBarrierSetC2State(Arena* arena) :

    50 int ZBarrierSetC2State::load_barrier_count() const {

    48     _stubs(new (arena) GrowableArray<ZLoadBarrierStubC2*>(arena, 8,  0, NULL)),

    51   return _load_barrier_nodes->length();

    49     _live(arena) {}

    52 }

    50 

    53 

    51   GrowableArray<ZLoadBarrierStubC2*>* stubs() {

    54 void ZBarrierSetC2State::add_load_barrier_node(LoadBarrierNode * n) {

    52     return _stubs;

    55   assert(!_load_barrier_nodes->contains(n), " duplicate entry in expand list");

    53   }

    56   _load_barrier_nodes->append(n);

    54 

    57 }

    55   RegMask* live(const Node* node) {

    58 

    56     if (!node->is_Mach()) {

    59 void ZBarrierSetC2State::remove_load_barrier_node(LoadBarrierNode * n) {

    57       // Don't need liveness for non-MachNodes

    60   // this function may be called twice for a node so check

    58       return NULL;

    61   // that the node is in the array before attempting to remove it

    59     }

    62   if (_load_barrier_nodes->contains(n)) {

    60 

    63     _load_barrier_nodes->remove(n);

    61     const MachNode* const mach = node->as_Mach();

    64   }

    62     if (mach->barrier_data() != ZLoadBarrierStrong &&

    65 }

    63         mach->barrier_data() != ZLoadBarrierWeak) {

    66 

    64       // Don't need liveness data for nodes without barriers

    67 LoadBarrierNode* ZBarrierSetC2State::load_barrier_node(int idx) const {

    65       return NULL;

    68   return _load_barrier_nodes->at(idx);

    66     }

    67 

    68     RegMask* live = (RegMask*)_live[node->_idx];

    69     if (live == NULL) {

    70       live = new (Compile::current()->comp_arena()->Amalloc_D(sizeof(RegMask))) RegMask();

    71       _live.map(node->_idx, (Node*)live);

    72     }

    73 

    74     return live;

    75   }

    76 };

    77 

    78 static ZBarrierSetC2State* barrier_set_state() {

    79   return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state());

    80 }

    81 

    82 ZLoadBarrierStubC2* ZLoadBarrierStubC2::create(const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) {

    83   ZLoadBarrierStubC2* const stub = new (Compile::current()->comp_arena()) ZLoadBarrierStubC2(node, ref_addr, ref, tmp, weak);

    84   if (!Compile::current()->in_scratch_emit_size()) {

    85     barrier_set_state()->stubs()->append(stub);

    86   }

    87 

    88   return stub;

    89 }

    90 

    91 ZLoadBarrierStubC2::ZLoadBarrierStubC2(const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) :

    92     _node(node),

    93     _ref_addr(ref_addr),

    94     _ref(ref),

    95     _tmp(tmp),

    96     _weak(weak),

    97     _entry(),

    98     _continuation() {

    99   assert_different_registers(ref, ref_addr.base());

   100   assert_different_registers(ref, ref_addr.index());

   101 }

   102 

   103 Address ZLoadBarrierStubC2::ref_addr() const {

   104   return _ref_addr;

   105 }

   106 

   107 Register ZLoadBarrierStubC2::ref() const {

   108   return _ref;

   109 }

   110 

   111 Register ZLoadBarrierStubC2::tmp() const {

   112   return _tmp;

   113 }

   114 

   115 address ZLoadBarrierStubC2::slow_path() const {

   116   const DecoratorSet decorators = _weak ? ON_WEAK_OOP_REF : ON_STRONG_OOP_REF;

   117   return ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators);

   118 }

   119 

   120 RegMask& ZLoadBarrierStubC2::live() const {

   121   return *barrier_set_state()->live(_node);

   122 }

   123 

   124 Label* ZLoadBarrierStubC2::entry() {

   125   // The _entry will never be bound when in_scratch_emit_size() is true.

   126   // However, we still need to return a label that is not bound now, but

   127   // will eventually be bound. Any lable will do, as it will only act as

   128   // a placeholder, so we return the _continuation label.

   129   return Compile::current()->in_scratch_emit_size() ? &_continuation : &_entry;

   130 }

   131 

   132 Label* ZLoadBarrierStubC2::continuation() {

   133   return &_continuation;

    72   return new(comp_arena) ZBarrierSetC2State(comp_arena);

   137   return new (comp_arena) ZBarrierSetC2State(comp_arena);

    73 }

   138 }

    74 

   139 

    75 ZBarrierSetC2State* ZBarrierSetC2::state() const {

   140 void ZBarrierSetC2::late_barrier_analysis() const {

    76   return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state());

   141   analyze_dominating_barriers();

    77 }

   142   compute_liveness_at_stubs();

    78 

   143 }

    79 bool ZBarrierSetC2::is_gc_barrier_node(Node* node) const {

   144 

    80   // 1. This step follows potential oop projections of a load barrier before expansion

   145 void ZBarrierSetC2::emit_stubs(CodeBuffer& cb) const {

    81   if (node->is_Proj()) {

   146   MacroAssembler masm(&cb);

    82     node = node->in(0);

   147   GrowableArray<ZLoadBarrierStubC2*>* const stubs = barrier_set_state()->stubs();

    83   }

   148 

    84 

   149   for (int i = 0; i < stubs->length(); i++) {

    85   // 2. This step checks for unexpanded load barriers

   150     // Make sure there is enough space in the code buffer

    86   if (node->is_LoadBarrier()) {

   151     if (cb.insts()->maybe_expand_to_ensure_remaining(Compile::MAX_inst_size) && cb.blob() == NULL) {

    87     return true;

   152       ciEnv::current()->record_failure("CodeCache is full");

    88   }

   153       return;

    89 

   154     }

    90   // 3. This step checks for the phi corresponding to an optimized load barrier expansion

   155 

    91   if (node->is_Phi()) {

   156     ZBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, stubs->at(i));

    92     PhiNode* phi = node->as_Phi();

   157   }

    93     Node* n = phi->in(1);

   158 

    94     if (n != NULL && n->is_LoadBarrierSlowReg()) {

   159   masm.flush();

    95       return true;

   160 }

    96     }

   161 

    97   }

   162 int ZBarrierSetC2::estimate_stub_size() const {

    98 

   163   Compile* const C = Compile::current();

    99   return false;

   164   BufferBlob* const blob = C->scratch_buffer_blob();

   100 }

   165   GrowableArray<ZLoadBarrierStubC2*>* const stubs = barrier_set_state()->stubs();

   101 

   166   int size = 0;

   102 void ZBarrierSetC2::register_potential_barrier_node(Node* node) const {

   167 

   103   if (node->is_LoadBarrier()) {

   168   for (int i = 0; i < stubs->length(); i++) {

   104     state()->add_load_barrier_node(node->as_LoadBarrier());

   169     CodeBuffer cb(blob->content_begin(), (address)C->scratch_locs_memory() - blob->content_begin());

   105   }

   170     MacroAssembler masm(&cb);

   106 }

   171     ZBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, stubs->at(i));

   107 

   172     size += cb.insts_size();

   108 void ZBarrierSetC2::unregister_potential_barrier_node(Node* node) const {

   173   }

   109   if (node->is_LoadBarrier()) {

   174 

   110     state()->remove_load_barrier_node(node->as_LoadBarrier());

   175   return size;

   111   }

   112 }

   113 

   114 void ZBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful, Compile* C) const {

   115   // Remove useless LoadBarrier nodes

   116   ZBarrierSetC2State* s = state();

   117   for (int i = s->load_barrier_count()-1; i >= 0; i--) {

   118     LoadBarrierNode* n = s->load_barrier_node(i);

   119     if (!useful.member(n)) {

   120       unregister_potential_barrier_node(n);

   121     }

   122   }

   123 }

   124 

   125 void ZBarrierSetC2::enqueue_useful_gc_barrier(PhaseIterGVN* igvn, Node* node) const {

   126   if (node->is_LoadBarrier() && !node->as_LoadBarrier()->has_true_uses()) {

   127     igvn->_worklist.push(node);

   128   }

   129 }

   130 

   131 const uint NoBarrier       = 0;

   132 const uint RequireBarrier  = 1;

   133 const uint WeakBarrier     = 2;

   134 const uint ExpandedBarrier = 4;

   135 

   136 static bool load_require_barrier(LoadNode* load)      { return (load->barrier_data() & RequireBarrier)  == RequireBarrier; }

   137 static bool load_has_weak_barrier(LoadNode* load)     { return (load->barrier_data() & WeakBarrier)     == WeakBarrier; }

   138 static bool load_has_expanded_barrier(LoadNode* load) { return (load->barrier_data() & ExpandedBarrier) == ExpandedBarrier; }

   139 static void load_set_expanded_barrier(LoadNode* load) { return load->set_barrier_data(ExpandedBarrier); }

   140 

   141 static void load_set_barrier(LoadNode* load, bool weak) {

   142   if (weak) {

   143     load->set_barrier_data(RequireBarrier | WeakBarrier);

   144   } else {

   145     load->set_barrier_data(RequireBarrier);

   146   }

   147 }

   148 

   149 // == LoadBarrierNode ==

   150 

   151 LoadBarrierNode::LoadBarrierNode(Compile* C,

   152                                  Node* c,

   153                                  Node* mem,

   154                                  Node* val,

   155                                  Node* adr,

   156                                  bool weak) :

   157     MultiNode(Number_of_Inputs),

   158     _weak(weak) {

   159   init_req(Control, c);

   160   init_req(Memory, mem);

   161   init_req(Oop, val);

   162   init_req(Address, adr);

   163   init_req(Similar, C->top());

   164 

   165   init_class_id(Class_LoadBarrier);

   166   BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();

   167   bs->register_potential_barrier_node(this);

   168 }

   169 

   170 uint LoadBarrierNode::size_of() const {

   171   return sizeof(*this);

   172 }

   173 

   174 bool LoadBarrierNode::cmp(const Node& n) const {

   175   ShouldNotReachHere();

   176   return false;

   177 }

   178 

   179 const Type *LoadBarrierNode::bottom_type() const {

   180   const Type** floadbarrier = (const Type **)(Compile::current()->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*)));

   181   Node* in_oop = in(Oop);

   182   floadbarrier[Control] = Type::CONTROL;

   183   floadbarrier[Memory] = Type::MEMORY;

   184   floadbarrier[Oop] = in_oop == NULL ? Type::TOP : in_oop->bottom_type();

   185   return TypeTuple::make(Number_of_Outputs, floadbarrier);

   186 }

   187 

   188 const TypePtr* LoadBarrierNode::adr_type() const {

   189   ShouldNotReachHere();

   190   return NULL;

   191 }

   192 

   193 const Type *LoadBarrierNode::Value(PhaseGVN *phase) const {

   194   const Type** floadbarrier = (const Type **)(phase->C->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*)));

   195   const Type* val_t = phase->type(in(Oop));

   196   floadbarrier[Control] = Type::CONTROL;

   197   floadbarrier[Memory]  = Type::MEMORY;

   198   floadbarrier[Oop]     = val_t;

   199   return TypeTuple::make(Number_of_Outputs, floadbarrier);

   200 }

   201 

   202 bool LoadBarrierNode::is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n) {

   203   if (phase != NULL) {

   204     return phase->is_dominator(d, n);

   205   }

   206 

   207   for (int i = 0; i < 10 && n != NULL; i++) {

   208     n = IfNode::up_one_dom(n, linear_only);

   209     if (n == d) {

   210       return true;

   211     }

   212   }

   213 

   214   return false;

   215 }

   216 

   217 LoadBarrierNode* LoadBarrierNode::has_dominating_barrier(PhaseIdealLoop* phase, bool linear_only, bool look_for_similar) {

   218   if (is_weak()) {

   219     // Weak barriers can't be eliminated

   220     return NULL;

   221   }

   222 

   223   Node* val = in(LoadBarrierNode::Oop);

   224   if (in(Similar)->is_Proj() && in(Similar)->in(0)->is_LoadBarrier()) {

   225     LoadBarrierNode* lb = in(Similar)->in(0)->as_LoadBarrier();

   226     assert(lb->in(Address) == in(Address), "");

   227     // Load barrier on Similar edge dominates so if it now has the Oop field it can replace this barrier.

   228     if (lb->in(Oop) == in(Oop)) {

   229       return lb;

   230     }

   231     // Follow chain of load barrier through Similar edges

   232     while (!lb->in(Similar)->is_top()) {

   233       lb = lb->in(Similar)->in(0)->as_LoadBarrier();

   234       assert(lb->in(Address) == in(Address), "");

   235     }

   236     if (lb != in(Similar)->in(0)) {

   237       return lb;

   238     }

   239   }

   240   for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {

   241     Node* u = val->fast_out(i);

   242     if (u != this && u->is_LoadBarrier() && u->in(Oop) == val && u->as_LoadBarrier()->has_true_uses()) {

   243       Node* this_ctrl = in(LoadBarrierNode::Control);

   244       Node* other_ctrl = u->in(LoadBarrierNode::Control);

   245       if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) {

   246         return u->as_LoadBarrier();

   247       }

   248     }

   249   }

   250 

   251   if (can_be_eliminated()) {

   252     return NULL;

   253   }

   254 

   255   if (!look_for_similar) {

   256     return NULL;

   257   }

   258 

   259   Node* addr = in(LoadBarrierNode::Address);

   260   for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) {

   261     Node* u = addr->fast_out(i);

   262     if (u != this && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) {

   263       Node* this_ctrl = in(LoadBarrierNode::Control);

   264       Node* other_ctrl = u->in(LoadBarrierNode::Control);

   265       if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) {

   266         ResourceMark rm;

   267         Unique_Node_List wq;

   268         wq.push(in(LoadBarrierNode::Control));

   269         bool ok = true;

   270         bool dom_found = false;

   271         for (uint next = 0; next < wq.size(); ++next) {

   272           Node *n = wq.at(next);

   273           if (n->is_top()) {

   274             return NULL;

   275           }

   276           assert(n->is_CFG(), "");

   277           if (n->is_SafePoint()) {

   278             ok = false;

   279             break;

   280           }

   281           if (n == u) {

   282             dom_found = true;

   283             continue;

   284           }

   285           if (n->is_Region()) {

   286             for (uint i = 1; i < n->req(); i++) {

   287               Node* m = n->in(i);

   288               if (m != NULL) {

   289                 wq.push(m);

   290               }

   291             }

   292           } else {

   293             Node* m = n->in(0);

   294             if (m != NULL) {

   295               wq.push(m);

   296             }

   297           }

   298         }

   299         if (ok) {

   300           assert(dom_found, "");

   301           return u->as_LoadBarrier();

   302         }

   303         break;

   304       }

   305     }

   306   }

   307 

   308   return NULL;

   309 }

   310 

   311 void LoadBarrierNode::push_dominated_barriers(PhaseIterGVN* igvn) const {

   312   // Change to that barrier may affect a dominated barrier so re-push those

   313   assert(!is_weak(), "sanity");

   314   Node* val = in(LoadBarrierNode::Oop);

   315 

   316   for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {

   317     Node* u = val->fast_out(i);

   318     if (u != this && u->is_LoadBarrier() && u->in(Oop) == val) {

   319       Node* this_ctrl = in(Control);

   320       Node* other_ctrl = u->in(Control);

   321       if (is_dominator(NULL, false, this_ctrl, other_ctrl)) {

   322         igvn->_worklist.push(u);

   323       }

   324     }

   325 

   326     Node* addr = in(LoadBarrierNode::Address);

   327     for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) {

   328       Node* u = addr->fast_out(i);

   329       if (u != this && u->is_LoadBarrier() && u->in(Similar)->is_top()) {

   330         Node* this_ctrl = in(Control);

   331         Node* other_ctrl = u->in(Control);

   332         if (is_dominator(NULL, false, this_ctrl, other_ctrl)) {

   333           igvn->_worklist.push(u);

   334         }

   335       }

   336     }

   337   }

   338 }

   339 

   340 Node *LoadBarrierNode::Identity(PhaseGVN *phase) {

   341   LoadBarrierNode* dominating_barrier = has_dominating_barrier(NULL, true, false);

   342   if (dominating_barrier != NULL) {

   343     assert(!is_weak(), "Weak barriers cant be eliminated");

   344     assert(dominating_barrier->in(Oop) == in(Oop), "");

   345     return dominating_barrier;

   346   }

   347 

   348   return this;

   349 }

   350 

   351 Node *LoadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) {

   352   if (remove_dead_region(phase, can_reshape)) {

   353     return this;

   354   }

   355 

   356   Node *val = in(Oop);

   357   Node *mem = in(Memory);

   358   Node *ctrl = in(Control);

   359 

   360   assert(val->Opcode() != Op_LoadN, "");

   361   assert(val->Opcode() != Op_DecodeN, "");

   362 

   363   if (mem->is_MergeMem()) {

   364     Node *new_mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw);

   365     set_req(Memory, new_mem);

   366     if (mem->outcnt() == 0 && can_reshape) {

   367       phase->is_IterGVN()->_worklist.push(mem);

   368     }

   369     return this;

   370   }

   371 

   372   LoadBarrierNode *dominating_barrier = NULL;

   373   if (!is_weak()) {

   374     dominating_barrier = has_dominating_barrier(NULL, !can_reshape, !phase->C->major_progress());

   375     if (dominating_barrier != NULL && dominating_barrier->in(Oop) != in(Oop)) {

   376       assert(in(Address) == dominating_barrier->in(Address), "");

   377       set_req(Similar, dominating_barrier->proj_out(Oop));

   378       return this;

   379     }

   380   }

   381 

   382   bool eliminate = can_reshape && (dominating_barrier != NULL || !has_true_uses());

   383   if (eliminate) {

   384     if (can_reshape) {

   385       PhaseIterGVN* igvn = phase->is_IterGVN();

   386       Node* out_ctrl = proj_out_or_null(Control);

   387       Node* out_res = proj_out_or_null(Oop);

   388 

   389       if (out_ctrl != NULL) {

   390         igvn->replace_node(out_ctrl, ctrl);

   391       }

   392 

   393       // That transformation may cause the Similar edge on the load barrier to be invalid

   394       fix_similar_in_uses(igvn);

   395       if (out_res != NULL) {

   396         if (dominating_barrier != NULL) {

   397           assert(!is_weak(), "Sanity");

   398           igvn->replace_node(out_res, dominating_barrier->proj_out(Oop));

   399         } else {

   400           igvn->replace_node(out_res, val);

   401         }

   402       }

   403     }

   404     return new ConINode(TypeInt::ZERO);

   405   }

   406 

   407   // If the Similar edge is no longer a load barrier, clear it

   408   Node* similar = in(Similar);

   409   if (!similar->is_top() && !(similar->is_Proj() && similar->in(0)->is_LoadBarrier())) {

   410     set_req(Similar, phase->C->top());

   411     return this;

   412   }

   413 

   414   if (can_reshape && !is_weak()) {

   415     // If this barrier is linked through the Similar edge by a

   416     // dominated barrier and both barriers have the same Oop field,

   417     // the dominated barrier can go away, so push it for reprocessing.

   418     // We also want to avoid a barrier to depend on another dominating

   419     // barrier through its Similar edge that itself depend on another

   420     // barrier through its Similar edge and rather have the first

   421     // depend on the third.

   422     PhaseIterGVN* igvn = phase->is_IterGVN();

   423     Node* out_res = proj_out(Oop);

   424     for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {

   425       Node* u = out_res->fast_out(i);

   426       if (u->is_LoadBarrier() && u->in(Similar) == out_res &&

   427           (u->in(Oop) == val || !u->in(Similar)->is_top())) {

   428         assert(!u->as_LoadBarrier()->is_weak(), "Sanity");

   429         igvn->_worklist.push(u);

   430       }

   431     }

   432     push_dominated_barriers(igvn);

   433   }

   434 

   435   return NULL;

   436 }

   437 

   438 uint LoadBarrierNode::match_edge(uint idx) const {

   439   ShouldNotReachHere();

   440   return 0;

   441 }

   442 

   443 void LoadBarrierNode::fix_similar_in_uses(PhaseIterGVN* igvn) {

   444   Node* out_res = proj_out_or_null(Oop);

   445   if (out_res == NULL) {

   446     return;

   447   }

   448 

   449   for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {

   450     Node* u = out_res->fast_out(i);

   451     if (u->is_LoadBarrier() && u->in(Similar) == out_res) {

   452       igvn->replace_input_of(u, Similar, igvn->C->top());

   453       --i;

   454       --imax;

   455     }

   456   }

   457 }

   458 

   459 bool LoadBarrierNode::has_true_uses() const {

   460   Node* out_res = proj_out_or_null(Oop);

   461   if (out_res != NULL) {

   462     for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {

   463       Node *u = out_res->fast_out(i);

   464       if (!u->is_LoadBarrier() || u->in(Similar) != out_res) {

   465         return true;

   466       }

   467     }

   468   }

   469   return false;

   477   Node* p = BarrierSetC2::load_at_resolved(access, val_type);

   183   Node* result = BarrierSetC2::load_at_resolved(access, val_type);

   478   if (!barrier_needed(access)) {

   184   if (barrier_needed(access) && access.raw_access()->is_Mem()) {

   479     return p;

   185     if ((access.decorators() & ON_WEAK_OOP_REF) != 0) {

   480   }

   186       access.raw_access()->as_Load()->set_barrier_data(ZLoadBarrierWeak);

   481 

   187     } else {

   482   bool weak = (access.decorators() & ON_WEAK_OOP_REF) != 0;

   188       access.raw_access()->as_Load()->set_barrier_data(ZLoadBarrierStrong);

   483   if (p->isa_Load()) {

   189     }

   484     load_set_barrier(p->as_Load(), weak);

   190   }

   485   }

   191 

   486   return p;

   192   return result;

   494     lsn->set_has_barrier();

   199     access.raw_access()->as_LoadStore()->set_barrier_data(ZLoadBarrierStrong);

   495   }

   200   }

   496   return lsn;

   201   return result;

   504     lsn->set_has_barrier();

   208     access.raw_access()->as_LoadStore()->set_barrier_data(ZLoadBarrierStrong);

   505   }

   209   }

   506   return lsn;

   210   return result;

   513     lsn->set_has_barrier();

   216     access.raw_access()->as_LoadStore()->set_barrier_data(ZLoadBarrierStrong);

   514   }

   217   }

   515   return lsn;

   218   return result;

   516 }

   219 }

   517 

   220 

   518 // == Macro Expansion ==

   221 bool ZBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type,

   519 

   222                                                     bool is_clone, ArrayCopyPhase phase) const {

   520 // Optimized, low spill, loadbarrier variant using stub specialized on register used

   223   return type == T_OBJECT || type == T_ARRAY;

   521 void ZBarrierSetC2::expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const {

   224 }

   522   PhaseIterGVN &igvn = phase->igvn();

   225 

   523   float unlikely  = PROB_UNLIKELY(0.999);

   226 // == Dominating barrier elision ==

   524 

   227 

   525   Node* in_ctrl = barrier->in(LoadBarrierNode::Control);

   228 static bool block_has_safepoint(const Block* block, uint from, uint to) {

   526   Node* in_mem = barrier->in(LoadBarrierNode::Memory);

   229   for (uint i = from; i < to; i++) {

   527   Node* in_val = barrier->in(LoadBarrierNode::Oop);

   230     if (block->get_node(i)->is_MachSafePoint()) {

   528   Node* in_adr = barrier->in(LoadBarrierNode::Address);

   231       // Safepoint found

   529 

   232       return true;

   530   Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);

   233     }

   531   Node* out_res = barrier->proj_out(LoadBarrierNode::Oop);

   234   }

   532 

   235 

   533   assert(barrier->in(LoadBarrierNode::Oop) != NULL, "oop to loadbarrier node cannot be null");

   236   // Safepoint not found

   534 

   237   return false;

   535   Node* jthread = igvn.transform(new ThreadLocalNode());

   238 }

   536   Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset()));

   239 

   537   Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr,

   240 static bool block_has_safepoint(const Block* block) {

   538                                                  TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(),

   241   return block_has_safepoint(block, 0, block->number_of_nodes());

   539                                                  MemNode::unordered));

   242 }

   540   Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val));

   243 

   541   Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask));

   244 static uint block_index(const Block* block, const Node* node) {

   542   Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type())));

   245   for (uint j = 0; j < block->number_of_nodes(); ++j) {

   543   Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();

   246     if (block->get_node(j) == node) {

   544   IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If();

   247       return j;

   545   Node* then = igvn.transform(new IfTrueNode(iff));

   248     }

   546   Node* elsen = igvn.transform(new IfFalseNode(iff));

   249   }

   547 

   250   ShouldNotReachHere();

   548   Node* new_loadp = igvn.transform(new LoadBarrierSlowRegNode(then, in_adr, in_val,

   251   return 0;

   549                                                               (const TypePtr*) in_val->bottom_type(), barrier->is_weak()));

   252 }

   550 

   253 

   551   // Create the final region/phi pair to converge cntl/data paths to downstream code

   254 void ZBarrierSetC2::analyze_dominating_barriers() const {

   552   Node* result_region = igvn.transform(new RegionNode(3));

   255   ResourceMark rm;

   553   result_region->set_req(1, then);

   256   Compile* const C = Compile::current();

   554   result_region->set_req(2, elsen);

   257   PhaseCFG* const cfg = C->cfg();

   555 

   258   Block_List worklist;

   556   Node* result_phi = igvn.transform(new PhiNode(result_region, TypeInstPtr::BOTTOM));

   259   Node_List mem_ops;

   557   result_phi->set_req(1, new_loadp);

   260   Node_List barrier_loads;

   558   result_phi->set_req(2, barrier->in(LoadBarrierNode::Oop));

   261 

   559 

   262   // Step 1 - Find accesses, and track them in lists

   560   igvn.replace_node(out_ctrl, result_region);

   263   for (uint i = 0; i < cfg->number_of_blocks(); ++i) {

   561   igvn.replace_node(out_res, result_phi);

   264     const Block* const block = cfg->get_block(i);

   562 

   265     for (uint j = 0; j < block->number_of_nodes(); ++j) {

   563   assert(barrier->outcnt() == 0,"LoadBarrier macro node has non-null outputs after expansion!");

   266       const Node* const node = block->get_node(j);

   564 

   267       if (!node->is_Mach()) {

   565   igvn.remove_dead_node(barrier);

   566   igvn.remove_dead_node(out_ctrl);

   567   igvn.remove_dead_node(out_res);

   568 

   569   assert(is_gc_barrier_node(result_phi), "sanity");

   570   assert(step_over_gc_barrier(result_phi) == in_val, "sanity");

   571 

   572   phase->C->print_method(PHASE_BARRIER_EXPANSION, 4, barrier->_idx);

   573 }

   574 

   575 bool ZBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const {

   576   ZBarrierSetC2State* s = state();

   577   if (s->load_barrier_count() > 0) {

   578     PhaseMacroExpand macro(igvn);

   579 

   580     int skipped = 0;

   581     while (s->load_barrier_count() > skipped) {

   582       int load_barrier_count = s->load_barrier_count();

   583       LoadBarrierNode * n = s->load_barrier_node(load_barrier_count-1-skipped);

   584       if (igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) {

   585         // Node is unreachable, so don't try to expand it

   586         s->remove_load_barrier_node(n);

   589       if (!n->can_be_eliminated()) {

   270 

   590         skipped++;

   271       MachNode* const mach = node->as_Mach();

   272       switch (mach->ideal_Opcode()) {

   273       case Op_LoadP:

   274       case Op_CompareAndExchangeP:

   275       case Op_CompareAndSwapP:

   276       case Op_GetAndSetP:

   277         if (mach->barrier_data() == ZLoadBarrierStrong) {

   278           barrier_loads.push(mach);

   279         }

   280       case Op_StoreP:

   281         mem_ops.push(mach);

   282         break;

   283 

   284       default:

   285         break;

   286       }

   287     }

   288   }

   289 

   290   // Step 2 - Find dominating accesses for each load

   291   for (uint i = 0; i < barrier_loads.size(); i++) {

   292     MachNode* const load = barrier_loads.at(i)->as_Mach();

   293     const TypePtr* load_adr_type = NULL;

   294     intptr_t load_offset = 0;

   295     const Node* const load_obj = load->get_base_and_disp(load_offset, load_adr_type);

   296     Block* const load_block = cfg->get_block_for_node(load);

   297     const uint load_index = block_index(load_block, load);

   298 

   299     for (uint j = 0; j < mem_ops.size(); j++) {

   300       MachNode* mem = mem_ops.at(j)->as_Mach();

   301       const TypePtr* mem_adr_type = NULL;

   302       intptr_t mem_offset = 0;

   303       const Node* mem_obj = mem_obj = mem->get_base_and_disp(mem_offset, mem_adr_type);

   304       Block* mem_block = cfg->get_block_for_node(mem);

   305       uint mem_index = block_index(mem_block, mem);

   306 

   307       if (load_obj == NodeSentinel || mem_obj == NodeSentinel ||

   308           load_obj == NULL || mem_obj == NULL ||

   309           load_offset < 0 || mem_offset < 0) {

   593       expand_loadbarrier_node(&macro, n);

   312 

   594       assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list");

   313       if (mem_obj != load_obj || mem_offset != load_offset) {

   595       if (C->failing()) {

   314         // Not the same addresses, not a candidate

   596         return true;

   315         continue;

   597       }

   316       }

   598     }

   317 

   599     while (s->load_barrier_count() > 0) {

   318       if (load_block == mem_block) {

   600       int load_barrier_count = s->load_barrier_count();

   319         // Earlier accesses in the same block

   601       LoadBarrierNode* n = s->load_barrier_node(load_barrier_count - 1);

   320         if (mem_index < load_index && !block_has_safepoint(mem_block, mem_index + 1, load_index)) {

   602       assert(!(igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())), "should have been processed already");

   321           load->set_barrier_data(ZLoadBarrierElided);

   603       assert(!n->can_be_eliminated(), "should have been processed already");

   322         }

   604       expand_loadbarrier_node(&macro, n);

   323       } else if (mem_block->dominates(load_block)) {

   605       assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list");

   324         // Dominating block? Look around for safepoints

   606       if (C->failing()) {

   325         ResourceMark rm;

   607         return true;

   326         Block_List stack;

   608       }

   327         VectorSet visited(Thread::current()->resource_area());

   609     }

   328         stack.push(load_block);

   610     igvn.set_delay_transform(false);

   329         bool safepoint_found = block_has_safepoint(load_block);

   611     igvn.optimize();

   330         while (!safepoint_found && stack.size() > 0) {

   612     if (C->failing()) {

   331           Block* block = stack.pop();

   613       return true;

   332           if (visited.test_set(block->_pre_order)) {

   614     }

   333             continue;

   615   }

   334           }

   616 

   335           if (block_has_safepoint(block)) {

   617   return false;

   336             safepoint_found = true;

   618 }

   337             break;

   619 

   338           }

   620 Node* ZBarrierSetC2::step_over_gc_barrier(Node* c) const {

   339           if (block == mem_block) {

   621   Node* node = c;

   340             continue;

   622 

   341           }

   623   // 1. This step follows potential oop projections of a load barrier before expansion

   342 

   624   if (node->is_Proj()) {

   343           // Push predecessor blocks

   625     node = node->in(0);

   344           for (uint p = 1; p < block->num_preds(); ++p) {

   626   }

   345             Block* pred = cfg->get_block_for_node(block->pred(p));

   627 

   346             stack.push(pred);

   628   // 2. This step checks for unexpanded load barriers

   347           }

   629   if (node->is_LoadBarrier()) {

   348         }

   630     return node->in(LoadBarrierNode::Oop);

   349 

   631   }

   350         if (!safepoint_found) {

   632 

   351           load->set_barrier_data(ZLoadBarrierElided);

   633   // 3. This step checks for the phi corresponding to an optimized load barrier expansion

   352         }

   634   if (node->is_Phi()) {

   353       }

   635     PhiNode* phi = node->as_Phi();

   354     }

   636     Node* n = phi->in(1);

   355   }

   637     if (n != NULL && n->is_LoadBarrierSlowReg()) {

   356 }

   638       assert(c == node, "projections from step 1 should only be seen before macro expansion");

   357 

   639       return phi->in(2);

   358 // == Reduced spilling optimization ==

   640     }

   359 

   641   }

   360 void ZBarrierSetC2::compute_liveness_at_stubs() const {

   642 

   643   return c;

   644 }

   645 

   646 Node* ZBarrierSetC2::step_over_gc_barrier_ctrl(Node* c) const {

   647   Node* node = c;

   648 

   649   // 1. This step follows potential ctrl projections of a load barrier before expansion

   650   if (node->is_Proj()) {

   651     node = node->in(0);

   652   }

   653 

   654   // 2. This step checks for unexpanded load barriers

   655   if (node->is_LoadBarrier()) {

   656     return node->in(LoadBarrierNode::Control);

   657   }

   658 

   659   return c;

   660 }

   661 

   662 bool ZBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const {

   663   return is_reference_type(type);

   664 }

   665 

   666 bool ZBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode) const {

   667   switch (opcode) {

   668     case Op_LoadBarrier:

   669       assert(0, "There should be no load barriers left");

   670     case Op_ZGetAndSetP:

   671     case Op_ZCompareAndExchangeP:

   672     case Op_ZCompareAndSwapP:

   673     case Op_ZWeakCompareAndSwapP:

   674 #ifdef ASSERT

   675       if (VerifyOptoOopOffsets) {

   676         MemNode *mem = n->as_Mem();

   677         // Check to see if address types have grounded out somehow.

   678         const TypeInstPtr *tp = mem->in(MemNode::Address)->bottom_type()->isa_instptr();

   679         ciInstanceKlass *k = tp->klass()->as_instance_klass();

   680         bool oop_offset_is_sane = k->contains_field_offset(tp->offset());

   681         assert(!tp || oop_offset_is_sane, "");

   682       }

   683 #endif

   684       return true;

   685     default:

   686       return false;

   687   }

   688 }

   689 

   690 bool ZBarrierSetC2::matcher_find_shared_visit(Matcher* matcher, Matcher::MStack& mstack, Node* n, uint opcode, bool& mem_op, int& mem_addr_idx) const {

   691   switch(opcode) {

   692     case Op_CallLeaf:

   693       if (n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr() ||

   694           n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr()) {

   695         mem_op = true;

   696         mem_addr_idx = TypeFunc::Parms + 1;

   697         return true;

   698       }

   699       return false;

   700     default:

   701       return false;

   702   }

   703 }

   704 

   705 bool ZBarrierSetC2::matcher_find_shared_post_visit(Matcher* matcher, Node* n, uint opcode) const {

   706   switch(opcode) {

   707     case Op_ZCompareAndExchangeP:

   708     case Op_ZCompareAndSwapP:

   709     case Op_ZWeakCompareAndSwapP: {

   710       Node *mem = n->in(MemNode::Address);

   711       Node *keepalive = n->in(5);

   712       Node *pair1 = new BinaryNode(mem, keepalive);

   713 

   714       Node *newval = n->in(MemNode::ValueIn);

   715       Node *oldval = n->in(LoadStoreConditionalNode::ExpectedIn);

   716       Node *pair2 = new BinaryNode(oldval, newval);

   717 

   718       n->set_req(MemNode::Address, pair1);

   719       n->set_req(MemNode::ValueIn, pair2);

   720       n->del_req(5);

   721       n->del_req(LoadStoreConditionalNode::ExpectedIn);

   722       return true;

   723     }

   724     case Op_ZGetAndSetP: {

   725       Node *keepalive = n->in(4);

   726       Node *newval = n->in(MemNode::ValueIn);

   727       Node *pair = new BinaryNode(newval, keepalive);

   728       n->set_req(MemNode::ValueIn, pair);

   729       n->del_req(4);

   730       return true;

   731     }

   732 

   733     default:

   734       return false;

   735   }

   736 }

   737 

   738 // == Verification ==

   739 

   740 #ifdef ASSERT

   741 

   742 static void verify_slippery_safepoints_internal(Node* ctrl) {

   743   // Given a CFG node, make sure it does not contain both safepoints and loads

   744   // that have expanded barriers.

   745   bool found_safepoint = false;

   746   bool found_load = false;

   747 

   748   for (DUIterator_Fast imax, i = ctrl->fast_outs(imax); i < imax; i++) {

   749     Node* node = ctrl->fast_out(i);

   750     if (node->in(0) != ctrl) {

   751       // Skip outgoing precedence edges from ctrl.

   752       continue;

   753     }

   754     if (node->is_SafePoint()) {

   755       found_safepoint = true;

   756     }

   757     if (node->is_Load() && load_require_barrier(node->as_Load()) &&

   758         load_has_expanded_barrier(node->as_Load())) {

   759       found_load = true;

   760     }

   761   }

   762   assert(!found_safepoint || !found_load, "found load and safepoint in same block");

   763 }

   764 

   765 static void verify_slippery_safepoints(Compile* C) {

   766   ResourceArea *area = Thread::current()->resource_area();

   767   Unique_Node_List visited(area);

   768   Unique_Node_List checked(area);

   769 

   770   // Recursively walk the graph.

   771   visited.push(C->root());

   772   while (visited.size() > 0) {

   773     Node* node = visited.pop();

   774 

   775     Node* ctrl = node;

   776     if (!node->is_CFG()) {

   777       ctrl = node->in(0);

   778     }

   779 

   780     if (ctrl != NULL && !checked.member(ctrl)) {

   781       // For each block found in the graph, verify that it does not

   782       // contain both a safepoint and a load requiring barriers.

   783       verify_slippery_safepoints_internal(ctrl);

   784 

   785       checked.push(ctrl);

   786     }

   787 

   788     checked.push(node);

   789 

   790     for (DUIterator_Fast imax, i = node->fast_outs(imax); i < imax; i++) {

   791       Node* use = node->fast_out(i);

   792       if (checked.member(use))  continue;

   793       if (visited.member(use))  continue;

   794       visited.push(use);

   795     }

   796   }

   797 }

   798 

   799 void ZBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const {

   800   switch(phase) {

   801     case BarrierSetC2::BeforeOptimize:

   802     case BarrierSetC2::BeforeLateInsertion:

   803       assert(state()->load_barrier_count() == 0, "No barriers inserted yet");

   804       break;

   805     case BarrierSetC2::BeforeMacroExpand:

   806       // Barrier placement should be set by now.

   807       verify_gc_barriers(false /*post_parse*/);

   808       break;

   809     case BarrierSetC2::BeforeCodeGen:

   810       // Barriers has been fully expanded.

   811       assert(state()->load_barrier_count() == 0, "No more macro barriers");

   812       verify_slippery_safepoints(compile);

   813       break;

   814     default:

   815       assert(0, "Phase without verification");

   816   }

   817 }

   818 

   819 // post_parse implies that there might be load barriers without uses after parsing

   820 // That only applies when adding barriers at parse time.

   821 void ZBarrierSetC2::verify_gc_barriers(bool post_parse) const {

   822   ZBarrierSetC2State* s = state();

   823   Compile* C = Compile::current();

   825   VectorSet visited(Thread::current()->resource_area());

   362   Compile* const C = Compile::current();

   826 

   363   Arena* const A = Thread::current()->resource_area();

   827   for (int i = 0; i < s->load_barrier_count(); i++) {

   364   PhaseCFG* const cfg = C->cfg();

   828     LoadBarrierNode* n = s->load_barrier_node(i);

   365   PhaseRegAlloc* const regalloc = C->regalloc();

   829 

   366   RegMask* const live = NEW_ARENA_ARRAY(A, RegMask, cfg->number_of_blocks() * sizeof(RegMask));

   830     // The dominating barrier on the same address if it exists and

   367   ZBarrierSetAssembler* const bs = ZBarrierSet::assembler();

   831     // this barrier must not be applied on the value from the same

   368   Block_List worklist;

   832     // load otherwise the value is not reloaded before it's used the

   369 

   833     // second time.

   370   for (uint i = 0; i < cfg->number_of_blocks(); ++i) {

   834     assert(n->in(LoadBarrierNode::Similar)->is_top() ||

   371     new ((void*)(live + i)) RegMask();

   835            (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() &&

   372     worklist.push(cfg->get_block(i));

   836             n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Address) == n->in(LoadBarrierNode::Address) &&

   373   }

   837             n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Oop) != n->in(LoadBarrierNode::Oop)),

   374 

   838            "broken similar edge");

   375   while (worklist.size() > 0) {

   839 

   376     const Block* const block = worklist.pop();

   840     assert(n->as_LoadBarrier()->has_true_uses(),

   377     RegMask& old_live = live[block->_pre_order];

   841            "found unneeded load barrier");

   378     RegMask new_live;

   842 

   379 

   843     // Several load barrier nodes chained through their Similar edge

   380     // Initialize to union of successors

   844     // break the code that remove the barriers in final graph reshape.

   381     for (uint i = 0; i < block->_num_succs; i++) {

   845     assert(n->in(LoadBarrierNode::Similar)->is_top() ||

   382       const uint succ_id = block->_succs[i]->_pre_order;

   846            (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() &&

   383       new_live.OR(live[succ_id]);

   847             n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Similar)->is_top()),

   384     }

   848            "chain of Similar load barriers");

   385 

   849 

   386     // Walk block backwards, computing liveness

   850     if (!n->in(LoadBarrierNode::Similar)->is_top()) {

   387     for (int i = block->number_of_nodes() - 1; i >= 0; --i) {

   851       ResourceMark rm;

   388       const Node* const node = block->get_node(i);

   852       Unique_Node_List wq;

   389 

   853       Node* other = n->in(LoadBarrierNode::Similar)->in(0);

   390       // Remove def bits

   854       wq.push(n);

   391       const OptoReg::Name first = bs->refine_register(node, regalloc->get_reg_first(node));

   855       for (uint next = 0; next < wq.size(); ++next) {

   392       const OptoReg::Name second = bs->refine_register(node, regalloc->get_reg_second(node));

   856         Node *nn = wq.at(next);

   393       if (first != OptoReg::Bad) {

   857         assert(nn->is_CFG(), "");

   394         new_live.Remove(first);

   858         assert(!nn->is_SafePoint(), "");

   395       }

   859 

   396       if (second != OptoReg::Bad) {

   860         if (nn == other) {

   397         new_live.Remove(second);

   861           continue;

   398       }

   862         }

   399 

   863 

   400       // Add use bits

   864         if (nn->is_Region()) {

   401       for (uint j = 1; j < node->req(); ++j) {

   865           for (uint i = 1; i < nn->req(); i++) {

   402         const Node* const use = node->in(j);

   866             Node* m = nn->in(i);

   403         const OptoReg::Name first = bs->refine_register(use, regalloc->get_reg_first(use));

   867             if (m != NULL) {

   404         const OptoReg::Name second = bs->refine_register(use, regalloc->get_reg_second(use));

   868               wq.push(m);

   405         if (first != OptoReg::Bad) {

   869             }

   406           new_live.Insert(first);

   870           }

   407         }

   871         } else {

   408         if (second != OptoReg::Bad) {

   872           Node* m = nn->in(0);

   409           new_live.Insert(second);

   873           if (m != NULL) {

   410         }

   874             wq.push(m);

   411       }

   875           }

   412 

   876         }

   413       // If this node tracks liveness, update it

   877       }

   414       RegMask* const regs = barrier_set_state()->live(node);

   878     }

   415       if (regs != NULL) {

   879   }

   416         regs->OR(new_live);

   880 }

   417       }

   881 

   418     }

   882 #endif // end verification code

   419 

   883 

   420     // Now at block top, see if we have any changes

   884 // If a call is the control, we actually want its control projection

   421     new_live.SUBTRACT(old_live);

   885 static Node* normalize_ctrl(Node* node) {

   422     if (new_live.is_NotEmpty()) {

   886  if (node->is_Call()) {

   423       // Liveness has refined, update and propagate to prior blocks

   887    node = node->as_Call()->proj_out(TypeFunc::Control);

   424       old_live.OR(new_live);

   888  }

   425       for (uint i = 1; i < block->num_preds(); ++i) {

   889  return node;

   426         Block* const pred = cfg->get_block_for_node(block->pred(i));

   890 }

   427         worklist.push(pred);

   891 

   428       }

   892 static Node* get_ctrl_normalized(PhaseIdealLoop *phase, Node* node) {

   429     }

   893   return normalize_ctrl(phase->get_ctrl(node));

   430   }

   894 }

   431 }

   895 

   896 static void call_catch_cleanup_one(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl);

   897 

   898 // This code is cloning all uses of a load that is between a call and the catch blocks,

   899 // to each use.

   900 

   901 static bool fixup_uses_in_catch(PhaseIdealLoop *phase, Node *start_ctrl, Node *node) {

   902 

   903   if (!phase->has_ctrl(node)) {

   904     // This node is floating - doesn't need to be cloned.

   905     assert(node != start_ctrl, "check");

   906     return false;

   907   }

   908 

   909   Node* ctrl = get_ctrl_normalized(phase, node);

   910   if (ctrl != start_ctrl) {

   911     // We are in a successor block - the node is ok.

   912     return false; // Unwind

   913   }

   914 

   915   // Process successor nodes

   916   int outcnt = node->outcnt();

   917   for (int i = 0; i < outcnt; i++) {

   918     Node* n = node->raw_out(0);

   919     assert(!n->is_LoadBarrier(), "Sanity");

   920     // Calling recursively, visiting leafs first

   921     fixup_uses_in_catch(phase, start_ctrl, n);

   922   }

   923 

   924   // Now all successors are outside

   925   // - Clone this node to both successors

   926   assert(!node->is_Store(), "Stores not expected here");

   927 

   928   // In some very rare cases a load that doesn't need a barrier will end up here

   929   // Treat it as a LoadP and the insertion of phis will be done correctly.

   930   if (node->is_Load()) {

   931     call_catch_cleanup_one(phase, node->as_Load(), phase->get_ctrl(node));

   932   } else {

   933     for (DUIterator_Fast jmax, i = node->fast_outs(jmax); i < jmax; i++) {

   934       Node* use = node->fast_out(i);

   935       Node* clone = node->clone();

   936       assert(clone->outcnt() == 0, "");

   937 

   938       assert(use->find_edge(node) != -1, "check");

   939       phase->igvn().rehash_node_delayed(use);

   940       use->replace_edge(node, clone);

   941 

   942       Node* new_ctrl;

   943       if (use->is_block_start()) {

   944         new_ctrl = use;

   945       } else if (use->is_CFG()) {

   946         new_ctrl = use->in(0);

   947         assert (new_ctrl != NULL, "");

   948       } else {

   949         new_ctrl = get_ctrl_normalized(phase, use);

   950       }

   951 

   952       phase->set_ctrl(clone, new_ctrl);

   953 

   954       if (phase->C->directive()->ZTraceLoadBarriersOption) tty->print_cr("  Clone op %i as %i to control %i", node->_idx, clone->_idx, new_ctrl->_idx);

   955       phase->igvn().register_new_node_with_optimizer(clone);

   956       --i, --jmax;

   957     }

   958     assert(node->outcnt() == 0, "must be empty now");

   959 

   960     // Node node is dead.

   961     phase->igvn().remove_dead_node(node);

   962   }

   963   return true; // unwind - return if a use was processed

   964 }

   965 

   966 // Clone a load to a specific catch_proj

   967 static Node* clone_load_to_catchproj(PhaseIdealLoop* phase, Node* load, Node* catch_proj) {

   968   Node* cloned_load = load->clone();

   969   cloned_load->set_req(0, catch_proj);      // set explicit control

   970   phase->set_ctrl(cloned_load, catch_proj); // update

   971   if (phase->C->directive()->ZTraceLoadBarriersOption) tty->print_cr("  Clone LOAD %i as %i to control %i", load->_idx, cloned_load->_idx, catch_proj->_idx);

   972   phase->igvn().register_new_node_with_optimizer(cloned_load);

   973   return cloned_load;

   974 }

   975 

   976 static Node* get_dominating_region(PhaseIdealLoop* phase, Node* node, Node* stop) {

   977   Node* region = node;

   978   while (!region->isa_Region()) {

   979     Node *up = phase->idom(region);

   980     assert(up != region, "Must not loop");

   981     assert(up != stop,   "Must not find original control");

   982     region = up;

   983   }

   984   return region;

   985 }

   986 

   987 // Clone this load to each catch block

   988 static void call_catch_cleanup_one(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl) {

   989   bool trace = phase->C->directive()->ZTraceLoadBarriersOption;

   990   phase->igvn().set_delay_transform(true);

   991 

   992   // Verify pre conditions

   993   assert(ctrl->isa_Proj() && ctrl->in(0)->isa_Call(), "Must be a call proj");

   994   assert(ctrl->raw_out(0)->isa_Catch(), "Must be a catch");

   995 

   996   if (ctrl->raw_out(0)->isa_Catch()->outcnt() == 1) {

   997     if (trace) tty->print_cr("Cleaning up catch: Skipping load %i, call with single catch", load->_idx);

   998     return;

   999   }

  1000 

  1001   // Process the loads successor nodes - if any is between

  1002   // the call and the catch blocks, they need to be cloned to.

  1003   // This is done recursively

  1004   for (uint i = 0; i < load->outcnt();) {

  1005     Node *n = load->raw_out(i);

  1006     assert(!n->is_LoadBarrier(), "Sanity");

  1007     if (!fixup_uses_in_catch(phase, ctrl, n)) {

  1008       // if no successor was cloned, progress to next out.

  1009       i++;

  1010     }

  1011   }

  1012 

  1013   // Now all the loads uses has been cloned down

  1014   // Only thing left is to clone the loads, but they must end up

  1015   // first in the catch blocks.

  1016 

  1017   // We clone the loads oo the catch blocks only when needed.

  1018   // An array is used to map the catch blocks to each lazily cloned load.

  1019   // In that way no extra unnecessary loads are cloned.

  1020 

  1021   // Any use dominated by original block must have an phi and a region added

  1022 

  1023   Node* catch_node = ctrl->raw_out(0);

  1024   int number_of_catch_projs = catch_node->outcnt();

  1025   Node** proj_to_load_mapping = NEW_RESOURCE_ARRAY(Node*, number_of_catch_projs);

  1026   Copy::zero_to_bytes(proj_to_load_mapping, sizeof(Node*) * number_of_catch_projs);

  1027 

  1028   // The phi_map is used to keep track of where phis have already been inserted

  1029   int phi_map_len = phase->C->unique();

  1030   Node** phi_map = NEW_RESOURCE_ARRAY(Node*, phi_map_len);

  1031   Copy::zero_to_bytes(phi_map, sizeof(Node*) * phi_map_len);

  1032 

  1033   for (unsigned int i = 0; i  < load->outcnt(); i++) {

  1034     Node* load_use_control = NULL;

  1035     Node* load_use = load->raw_out(i);

  1036 

  1037     if (phase->has_ctrl(load_use)) {

  1038       load_use_control = get_ctrl_normalized(phase, load_use);

  1039       assert(load_use_control != ctrl, "sanity");

  1040     } else {

  1041       load_use_control = load_use->in(0);

  1042     }

  1043     assert(load_use_control != NULL, "sanity");